diff --git a/.dependency-cruiser.cjs b/.dependency-cruiser.cjs index ffb6839..5593765 100644 --- a/.dependency-cruiser.cjs +++ b/.dependency-cruiser.cjs @@ -1,7 +1,13 @@ const FLOWSAFE_PUBLIC_ENTRY = '^packages/flowsafe/src/(?:index|host-kit/index|agent-runner/index|signals/client)\\.ts$'; +// `principal-identity` is the import-free half of `principal` (the kind list +// and the two identity predicates). It is admitted here for the same reason the +// others are, and is strictly leafier than any of them: it imports nothing at +// all, so it cannot widen what do-runner reaches through approval-api. const ALLOWED_APPROVAL_API_LEAVES = - '^packages/flowsafe/src/approval-api/(?:principal|contract|types)\\.ts$'; + '^packages/flowsafe/src/approval-api/(?:principal-identity|principal|contract|types)\\.ts$'; +// NOT extended with `principal-identity`: this is the exception list for the +// one tolerated import cycle, and a module with no imports can never be in one. const KNOWN_APPROVAL_API_CYCLE = '^packages/flowsafe/src/approval-api/(?:principal|contract|types)\\.ts$'; @@ -44,7 +50,7 @@ module.exports = { name: 'do-runner-approval-api-leaves-only', severity: 'error', comment: - 'do-runner may reach approval-api only through principal.ts, contract.ts, and their type-only types.ts leaf.', + 'do-runner may reach approval-api only through principal.ts, its import-free principal-identity.ts half, contract.ts, and their type-only types.ts leaf.', from: { path: [ '^packages/flowsafe/src/do-runner/index\\.ts$', diff --git a/docs/api-reference.md b/docs/api-reference.md index 6af7586..200bdc6 100644 --- a/docs/api-reference.md +++ b/docs/api-reference.md @@ -39,19 +39,41 @@ New host-side and React features remain subpath-only so importing the root does | `@proofoftech/flowsafe/agent-host` | Server-only guarded-agent catalogs, authenticated start/status/NDJSON routes, thread hosting, and approval-only resume | | `@proofoftech/flowsafe/agent-runner` | Runtime-driven Mastra durable agents, approval suspend parsing, and restart resume | | `@proofoftech/flowsafe/approval-api` | Records, actor resolver, deployment store, service, router, grant provider, retention, SLA, notifications, and stream events | -| `@proofoftech/flowsafe/background-tasks` | D1 task domains, deployment task host, routes, and terminal-task purge | +| `@proofoftech/flowsafe/background-tasks` | D1 task domains, deployment task host, fence-suspension marker, routes, and terminal-task purge | | `@proofoftech/flowsafe/approval-ui` | React dashboard, client, headless hook, component slots, and live transport | | `@proofoftech/flowsafe/artifacts` | R2 artifact store and in-memory bucket | | `@proofoftech/flowsafe/audit-export` | Queue producer sink and NDJSON SIEM consumer | -| `@proofoftech/flowsafe/do-runner` | Runtime, Durable Object classes, D1 storage, deployment sentinel and caller attestation, identity helpers, pub/sub, retention, and run summaries | +| `@proofoftech/flowsafe/do-runner` | Runtime, Durable Object classes, D1 storage, deployment sentinel and caller attestation, identity helpers, pub/sub, retention, run summaries, execution fence, start reservations, and drain inventory | | `@proofoftech/flowsafe/goals` | Objective HTTP router and goal request-context contract | -| `@proofoftech/flowsafe/host-kit` | Authenticator and verifier seams, run/thread/hub/provider topologies, routes, approval bridges, tickets, and composed Worker | +| `@proofoftech/flowsafe/host-kit` | Authenticator and verifier seams, run/thread/hub/provider topologies, routes, approval bridges, tickets, composed Worker, and execution-fence and inventory admin routes | | `@proofoftech/flowsafe/host-kit/module` | Workflow-module interface for import-safe host registration | | `@proofoftech/flowsafe/schedules` | D1 schedule domain, deployment router, reserved-context guard, and CAS tick | | `@proofoftech/flowsafe/signal-providers` | Provider adapters, host Durable Object, topology, subscriptions, verified webhooks, and GitHub provider | | `@proofoftech/flowsafe/signals` | D1 signal domains, thread routes, canonical content-policy seam, ingress router, notification dispatch, and client | | `@proofoftech/flowsafe/signals/client` | DOM-free `SignalClient` without host-side signal code | +The migration and idempotency surfaces are grouped by subpath: + +| Import path | Operational exports | +| --- | --- | +| `@proofoftech/flowsafe/do-runner` | `ExecutionFenceStore`, `ExecutionFenceWiring`, `ExecutionFenceState`, `ExecutionFenceReading`, `ExecutionFenceTransition`, `executionFenceFor`, `readExecutionFence`, `admitsRunStart`, `admitsExistingRun`, `admitsWorkAuthoring`, `admitsDrainableExecution`, `ExecutionFencedError`, `ExecutionFenceUnreadableError`, `FenceTransitionConflictError`, `StartIdempotencyStore`, `StartIdempotencyWiring`, `startIdempotencyFor`, `beginIdempotentStart`, the start-reservation error classes and guards, `DeploymentInventory`, `INVENTORY_CATEGORIES`, `INVENTORY_DRAIN_PROOF`, `INVENTORY_UNENUMERABLE`, and `FLOWSAFE_TABLES` | +| `@proofoftech/flowsafe/host-kit` | `RunRouterOptions`, `RunRouterStartIdempotency`, `createRunRouter`, `FlowsafeWorkerEnv`, and `createFlowsafeWorker`; the composed Worker owns the execution-fence and inventory admin routes | +| `@proofoftech/flowsafe/agent-host` | `AgentThreadStartInput`, `AgentThreadTopologyOptions`, `AgentThreadTopology`, and `createAgentThreadTopology` | +| `@proofoftech/flowsafe/background-tasks` | `BackgroundTaskHost`, `BackgroundTaskReads`, and `EXECUTION_FENCE_SUSPEND_KEY` | +| `@proofoftech/flowsafe/deployment-identity-protocol` | `EXECUTION_FENCE_TABLE`, `EXECUTION_FENCE_ROW_ID`, `EXECUTION_FENCE_STATES`, `INITIAL_EXECUTION_FENCE_STATES`, `EXECUTION_FENCE_DDL`, `InitialExecutionFenceState`, `assertInitialExecutionFenceState`, and `provisionDeploymentIdentityProtocol` | + +## fleet-control root export + +`@proofoftech/fleet-control` exposes trusted provisioning and fleet lifecycle operations from its root: + +| Surface | Main exports | +| --- | --- | +| Provisioning | `provisionDeployment`, `cleanupDeploymentArtifacts`, `decommissionDeployment`, `forceDecommissionDeployment`, `ProvisionDeploymentOptions`, `ProvisioningBackend`, `PlainWorkerRouteApi`, and `SeedDeploymentIdentityOptions` | +| Fleet lifecycle | `migrateFleet`, `rollbackExternalRelease`, `auditFleetDrift`, `fleetVersionReport`, `FleetRecord`, and `D1FleetStateStore` | +| Active-route attestation | `attestFleetRecordActiveRoute`, `attestConvergedActiveRoute`, `ActiveRouteAttestation`, `ActiveRouteAttestationError`, `ActiveRouteExpectation`, `AttestConvergedActiveRouteOptions`, and `ObservedActiveRoute` | +| Settlement | `fleetSettlementKey`, `FleetSettlementContext`, `FleetSettlementEntry`, and `FleetSettlementHost` | +| Backends and provider client | `WranglerLoopBackend`, `WorkersForPlatformsBackend`, `CloudflareProvisioningClient`, `D1CloudflareApiRateCoordinator`, and `ProcessLocalCloudflareApiRateCoordinator` | + ## Browser and server boundaries Safe browser imports: diff --git a/docs/deployment-reference.md b/docs/deployment-reference.md index fcb25c1..0a64639 100644 --- a/docs/deployment-reference.md +++ b/docs/deployment-reference.md @@ -30,7 +30,11 @@ Do not perform a physical-isolation cutover as an in-place update of a pooled Wo | `MAINTENANCE` | Durable Object namespace | Fixed deployment singleton for deadline expiry, SLA sweep, retention purge, and optional schedule tick | | `MAINTENANCE_ADMIN_SECRET` | Secret | Control-plane credential for maintenance bootstrap and status; distinct from deployment identity | -Provision the sentinel before application migrations or traffic. Install host-provided Wrangler `>=4.118 <5` in the application, run `npx flowsafe-provision --database --tag --remote --config wrangler.jsonc`, then set distinct `DEPLOYMENT_IDENTITY_SECRET` and `MAINTENANCE_ADMIN_SECRET` values with `wrangler secret put`. Wrangler is not installed as a Flowsafe peer. The CLI is published with Flowsafe. It verifies the exact singleton schema, refuses to re-home an owned database, and refuses to adopt an unowned database that already contains application tables. +Provision the sentinel before application migrations or traffic. Install host-provided Wrangler `>=4.118 <5` in the application, then run `npx flowsafe-provision --database --tag --initial-fence-state --remote --config wrangler.jsonc`. Set distinct `DEPLOYMENT_IDENTITY_SECRET` and `MAINTENANCE_ADMIN_SECRET` values with `wrangler secret put`. Wrangler is not installed as a Flowsafe peer. The CLI is published with Flowsafe. It verifies the exact singleton schema, refuses to re-home an owned database, and refuses to adopt an unowned database that already contains application tables. + +The initial fence state is required and has no default. Choose `open` for an ordinary deployment or `migration-locked` for a deployment that must remain inert until a migration completes. A database created before Flowsafe 0.20 can lack the fence table or row; runtime reads treat that absence as `open` for upgrade compatibility. + +Treat `migration-locked` at birth as a verified postcondition. After provisioning, authenticate `GET /admin/execution-fence` and fail the provisioning operation unless the response state is `migration-locked`. This check also makes version skew loud: an older control plane that seeds a 0.20 database without establishing the required state cannot pass the postcondition. The Worker compares `DEPLOYMENT_TENANT` with `flowsafe_deployment.tenant_tag`. Worker topologies stamp the internal credential on every Durable Object fetch, and the target compares it in constant time before reading storage. This additional caller check prevents an external or cross-script namespace binding from reaching another deployment's objects. Alarms validate the target environment and sentinel because they have no caller request. @@ -90,11 +94,12 @@ Host routing belongs to the provisioning control plane. It must resolve a hostna | --- | --- | --- | | `DEPLOYMENT_TENANT` | None | Required provisioning tag. Protected routes return `503` unless it matches the D1 sentinel | | `DEPLOYMENT_IDENTITY_SECRET` | None | Required internal credential. Worker-to-Durable-Object requests fail before storage unless it matches | -| `MAINTENANCE_ADMIN_SECRET` | None | Required control-plane credential. Maintenance admin routes return `503` when absent or malformed | +| `MAINTENANCE_ADMIN_SECRET` | None | Shared-secret credential. Execution-fence and inventory routes always return `503` when it is absent or malformed. Ensure-maintenance and maintenance-status instead accept a relayed fleet capability when `FLEET_MAINTENANCE_CAPABILITIES=required` and the secret is absent | | `APPROVAL_ACTOR_TOKENS` | Empty | Static verifier map. Empty means every authenticated route returns 401 | | `APPROVAL_SLA_SECONDS` | `14400` | SLA assigned to new approval records | | `APPROVAL_ALLOW_SELF_DECISION` | Unset | Separation of duties enabled. Accepts `true` or a comma-separated role list | | `RUN_RETENTION_DAYS` | `30` | Age for terminal workflow snapshot purge; `0` means immediate eligibility | +| `START_IDEMPOTENCY_RETENTION_DAYS` | `RUN_RETENTION_DAYS` | Age for terminal start-reservation purge; keep it at least as long as callers may retry a key | | `APPROVAL_RETENTION_DAYS` | `30` | Age for approved/rejected approval purge | | `THREAD_RETENTION_DAYS` | Unset | Idle thread and message purge. Unset keeps conversations | @@ -136,6 +141,28 @@ GET /api/stream/run/:workflowId/:runId The stream routes mount only when streaming is configured. The approval create route is off unless the host explicitly enables its capability-free form. +### Control-plane routes + +The composed Worker mounts operational routes before tenant routers: + +```text +GET /admin/execution-fence +POST /admin/execution-fence +GET /admin/inventory +POST /admin/ensure-maintenance +GET /admin/maintenance-status +``` + +The execution-fence and inventory routes require `Authorization: Bearer `. `MAINTENANCE_ADMIN_SECRET` must contain 32 to 256 visible ASCII characters and must differ from `DEPLOYMENT_IDENTITY_SECRET`. If the secret is absent, both routes return `503`; they never delegate authentication to a fleet capability. + +The ensure-maintenance and maintenance-status routes use the same shared-secret rule when `MAINTENANCE_ADMIN_SECRET` is configured. When `FLEET_MAINTENANCE_CAPABILITIES=required` and that secret is absent, they instead relay the caller's Ed25519 fleet capability token for downstream verification. The Worker does not compare the relayed token with a shared secret, and caps the credential at 2,048 characters. + +The deployment-identity gate runs before every control-plane route, so a binding or sentinel mismatch still returns `503` before administration. + +`GET /admin/execution-fence` returns `{ state, proofKey?, proofRunId? }`. `POST /admin/execution-fence` accepts `{ expected, next, proofKey? }` and applies one CAS transition. A stale `expected` value returns `409` with `reason.code: 'FENCE_CAS_CONFLICT'` and the current state. The host owns transition policy; Flowsafe validates only the state vocabulary, CAS, and proof-key shape. + +`GET /admin/inventory` returns the category index. Add `?category=&cursor=&limit=` to page one category. Prove a drain only from `draining`: sweep every work category to empty twice, at least 60 seconds apart. Standing categories remain present by design, and persisted idle signals deliberately carry across the migration. + ### Advanced routes ```text @@ -226,6 +253,8 @@ TTL retention, authorized domain deletion, and deployment decommissioning are se | Schedule triggers | Opt-in fire-history TTL | | Background tasks | Terminal-state TTL | | Provider subscriptions | No TTL; authorized deletion or deployment decommissioning | +| `flowsafe_execution_fence` | Singleton deployment control row; no TTL. An absent pre-0.20 row reads as `open` | +| `flowsafe_start_idempotency` | Terminal reservations remain for at least the run-summary horizon, then purge with run retention | | `flowsafe_resource_owners` | Run retention and schedule deletion release their claims. Thread and resource claims require explicit host teardown or deployment decommissioning | | R2 artifacts | Delete with the owning snapshot purge and deployment decommissioning | diff --git a/docs/do-runner-design.md b/docs/do-runner-design.md index 89c2c84..2482c65 100644 --- a/docs/do-runner-design.md +++ b/docs/do-runner-design.md @@ -109,6 +109,45 @@ Terminate: A repeated request reads the terminal snapshot and returns the same summary. After ownership release, the public router delegates replay authorization to the owner object. The object accepts only a principal recorded by the original transition. +### Fence execution during a deployment migration + +The execution fence controls one physical deployment, which is also one tenant boundary. It is never scoped to an actor or run. Each admission reads the current row without memoization, and storage failures fail closed. + +The four states apply this matrix: + +| Entry family | `open` | `draining` | `migration-locked` | `proof-only` | +| --- | --- | --- | --- | --- | +| New run start | Admit | Refuse | Refuse | Admit only when `idempotencyKey` matches `proofKey` | +| Resume, approval decision, or delivery to an existing run | Admit | Admit | Refuse | Admit only for the bound `proofRunId` | +| Schedule or objective authoring and due-fire claims | Admit | Refuse | Refuse | Refuse | +| Background tasks: new enqueue | Admit | Admit | Refuse | Refuse | +| Background tasks: dispatch or stale re-drive | Admit | Admit | Refuse | Refuse | +| Notification dispatch, provider polling, and webhook ingress | Admit | Admit | Refuse | Refuse | + +Fence-parked task rows are still swept for resume under the lock and simply re-park. This churn is self-limiting, and no task body executes. + +Schedule pause and delete, objective clear, reads, termination, cancellation, and timeout remain available in every state because they remove work or observe state. During `draining`, signal wakes that would mint a run degrade to persistence, so the next deployment can deliver them. The fence never preempts compute already in flight. Drain first, prove the work inventory empty, and only then transition to `migration-locked`. + +A refused request returns `503` with `reason.code: 'EXECUTION_FENCED'` and the current state. `proof-only` admits only proof-bound operations: webhook and poll ingress cannot identify the target run and remain blocked. The package enforces state names, compare-and-set (CAS), and proof-key requirements, while the host owns transition policy. + +### Start a run idempotently + +An idempotent start stores a caller-supplied `idempotencyKey` before execution and binds it to one server-minted run ID. Callers never supply a run ID. `POST /runs` continues to return `400` when its body contains `runId`; the key is the request handle, not the run identity. + +The key is accepted by `POST /runs`, trusted `AgentThreadStartInput` calls, and the internal `streamUntilPersisted()` start. `RunRouterOptions`, `AgentThreadTopologyOptions`, and `StorageInitOptions` require `startIdempotency` wiring. Pass `'none'` only when the host has no reservation database; a keyed start on that host returns `IDEMPOTENT_START_UNSUPPORTED`. + +Retries use these outcomes: + +- A persisted snapshot returns the same run and state without executing again +- A `reserved` row with no snapshot reclaims the same reserved run ID +- A live `started` row with no snapshot returns `IDEMPOTENT_START_PENDING` and `pendingSince` +- A non-live `started` row with no snapshot returns `IDEMPOTENT_START_UNRESOLVABLE` +- A `terminal` row whose snapshot expired returns `IDEMPOTENT_START_ALREADY_SETTLED` + +`IDEMPOTENT_START_UNRESOLVABLE` is a point-in-time probe. A read can occur between the Worker-side claim and the run object learning that execution started, so re-probe before investigating or choosing a fresh key. Flowsafe never starts another run automatically. A replayed suspended start returns the persisted `RunSummary` without the start response's `approval` and `approvals` fields; read `GET /runs/:workflowId/:runId` to reconcile approval state. + +The reservation remains valid for its retention horizon. Once the terminal reservation is purged, the same key is fresh and can start another run. Keep the horizon at least as long as callers can retry. + ## Durable state ### D1 snapshot @@ -123,6 +162,12 @@ Mastra's `mastra_workflow_snapshot` row is authoritative for: Flowsafe does not maintain a parallel custom workflow state object. +### Execution fence and start reservations + +`flowsafe_execution_fence` stores the deployment's singleton fence state, optional proof key, and optional bound proof run. State transitions compare the caller's `expected` state before they write. A database created by Flowsafe 0.19 has no row or table, which reads as `open`; provisioning from 0.20 onward writes an explicit initial row. + +`flowsafe_start_idempotency` stores owner, target, server-minted run ID, reservation state, and timestamps. The claim from `reserved` to `started` is the cross-isolate serializer. Terminal run cleanup pairs snapshot and reservation retention so a spent key remains distinguishable from a fresh key until its configured horizon expires. + ### Snapshot provenance Flowsafe stores trusted run provenance under reserved request-context keys in the same authoritative snapshot as the workflow state. Provenance records the initiating actor, a per-leg attempt token, and a monotonic ordinal per run and step: @@ -257,6 +302,26 @@ POST /runs/:workflowId/:runId/terminate-replay POST /runs/:workflowId/:runId/deadline ``` +The composed Worker also exposes these control-plane routes before tenant routers: + +```text +GET /admin/execution-fence +POST /admin/execution-fence +GET /admin/inventory +``` + +All three require a bearer token matching `MAINTENANCE_ADMIN_SECRET`, which must differ from `DEPLOYMENT_IDENTITY_SECRET`. The deployment-identity gate still runs first. A mis-provisioned deployment therefore returns `503` before an operator can read or move its fence. + +`GET /admin/execution-fence` returns `{ state, proofKey?, proofRunId? }`. `POST /admin/execution-fence` accepts `{ expected, next, proofKey? }`; a stale expectation returns `409` with `reason.code: 'FENCE_CAS_CONFLICT'` and the current state. + +`GET /admin/inventory` returns an index or one keyset-paginated category selected with `?category&cursor&limit`. The work categories are `runs`, `approvals-waiting`, `schedule-deferred-dispatches`, `pending-notifications`, `background-tasks`, `resource-owners`, and `start-reservations`. The standing categories are `schedules` and `signal-subscriptions`; they are reported for reconciliation and never required to empty. + +`INVENTORY_DRAIN_PROOF` defines a proof as two consecutive full sweeps with every work category empty, at least 60 seconds apart, while the fence remains `draining`. Each reading is a point-in-time observation rather than a snapshot and can move in either direction because draining still admits work. Empty results cannot over-count, and keyset pagination never skips a row that existed before the sweep began. + +If a host needs a hard guarantee, it can re-sweep once after transitioning to `migration-locked`. An empty post-lock sweep is conclusive. A non-empty sweep means work is still outstanding: either it entered after the proof or the lock parked it before it finished. Return to `draining`, let it finish, and repeat the proof before locking again. An inventory read taken under `migration-locked` measures what the fence parked rather than what the deployment would otherwise be doing. + +`INVENTORY_UNENUMERABLE` declares two visibility gaps instead of hiding them. The run-owner recovery journal can remain outside D1 for one 60-second alarm cadence. Persisted idle signals have no enumerable consumption marker and deliberately survive the migration. `FLOWSAFE_TABLES` accounts for every Flowsafe-owned table, including justified exclusions, so a new table cannot silently escape the inventory census. + The public Worker normally exposes its own authenticated route facade and forwards to these internal routes. A stream request requires a WebSocket upgrade and workerd's hibernatable socket API; otherwise it returns 426 and the client polls status. The raw resume surface carries no approval grant. A protected connector still requires a matching stored decision. @@ -291,14 +356,27 @@ Runtime-derived base keys win over stored or client-provided context. `breakwate ## Error taxonomy -The runtime distinguishes: - -- unknown workflow; -- unknown run; -- duplicate run; -- run not suspended; -- client-fixable input, resume-data, or step errors; -- internal execution or storage errors. +The runner preserves stable statuses and structured refusal reasons across Durable Object and host-router boundaries: + +| Error | HTTP status | `reason.code` | Meaning | +| --- | --- | --- | --- | +| `ExecutionFencedError` | `503` | `EXECUTION_FENCED` | The current state refuses this execution entry | +| `ExecutionFenceUnreadableError` | `503` | `EXECUTION_FENCE_UNREADABLE` | Storage did not provide a trustworthy fence state | +| `FenceTransitionConflictError` | `409` | `FENCE_CAS_CONFLICT` | The caller's expected state is stale | +| `InvalidExecutionFenceRequestError` | `400` | `INVALID_EXECUTION_FENCE_REQUEST` | A transition body contains an invalid state or proof key | +| `StartReservationOwnerMismatchError` | `403` | `IDEMPOTENT_START_OWNER_MISMATCH` | Another principal owns the key | +| `StartReservationTargetMismatchError` | `409` | `IDEMPOTENT_START_TARGET_MISMATCH` | The caller's key names another workflow or agent | +| `IdempotentStartPendingError` | `503` | `IDEMPOTENT_START_PENDING` | The keyed run is live but has no persisted summary yet | +| `IdempotentStartUnresolvableError` | `409` | `IDEMPOTENT_START_UNRESOLVABLE` | No snapshot or live execution currently resolves the claimed run | +| `IdempotentStartAlreadySettledError` | `409` | `IDEMPOTENT_START_ALREADY_SETTLED` | The run settled and its summary expired | +| `StartIdempotencyUnsupportedError` | `503` | `IDEMPOTENT_START_UNSUPPORTED` | The host did not wire reservation storage | +| `StartReservationUnreadableError` | `503` | `IDEMPOTENT_START_UNREADABLE` | The reservation store or row is unreadable | +| `InvalidStartIdempotencyRequestError` | `400` | `INVALID_START_IDEMPOTENCY_REQUEST` | The key or reservation request is malformed | +| `InvalidInventoryRequestError` | `400` | `INVALID_INVENTORY_REQUEST` | The category, cursor, or limit is invalid | + +`isStartReservationRefusal()` recognizes the five reservation decisions, unsupported wiring, and malformed input. It excludes `StartReservationUnreadableError`, which propagates as an operational storage failure. + +The runtime also distinguishes unknown workflows, unknown runs, duplicate runs, runs that are not suspended, client-fixable input or resume-data errors, and internal execution or storage failures. The Durable Object maps known errors to stable HTTP status codes through `doErrorResponse()`. Unknown failures return an internal error without copying arbitrary thrown data to an audit sink. diff --git a/docs/fleet-control.md b/docs/fleet-control.md index 7c7e42d..16b2e24 100644 --- a/docs/fleet-control.md +++ b/docs/fleet-control.md @@ -38,6 +38,7 @@ const result = await provisionDeployment({ store, spec: deployment, secrets: generateDeploymentSecrets(), + initialExecutionFenceState: 'open', }); if (!result.maintenance.armed) { @@ -49,6 +50,8 @@ The function persists every completed phase and validates the immutable tenant, Provisioning resumes from its last durable phase without repeating a committed step. Before `ready`, it compares the exact live tenant, environment, D1 binding, schema, specification digest, Durable Object bindings, plain-text variables, and secret names. Plain Worker, dispatch Worker, backend-switch, and control-plane inspection must consume every raw provider binding entry. An unknown type, malformed entry, duplicate name, binding absent from the structured inspection, or missing complete inventory fails closed even when the desired application groups are empty. Ordinary Worker secret names come from the authoritative secret-list API; if version resources also report them, the two inventories must agree. A failed first create revokes credentials and removes resources created by that attempt. If an upload may have succeeded, cleanup treats the Worker as present until deletion is positively confirmed; D1 is never deleted while a Worker or route may remain. Cleanup errors remain attached to `ProvisioningError.cleanupErrors`, and the durable phase remains available to retry or to `cleanupDeploymentArtifacts()`. +`initialExecutionFenceState` is required and accepts `open` or `migration-locked`. It is a provisioning decision, not part of `DeploymentSpec`, so it does not alter the specification digest. `provisionDeployment()` is asynchronous from entry: invalid initial state and other validation failures reject its promise instead of throwing before a promise exists. The final ready record uses the artifact version from post-promotion route attestation rather than the candidate inspection. + ## Define application bindings `DeploymentSpec.application` declares three canonical binding groups: @@ -99,6 +102,43 @@ External candidates cannot own Durable Object classes or migrations, and an exte Supply a backend-owned `platformProfileFor(spec)` provider when the Workers for Platforms backend can receive external artifacts. The provider returns the trusted state artifact, platform state migration history, organization egress allowlist, and optional legacy bridge template. Fleet control validates and hashes that profile before upload. A profile or policy change requires `migrateFleet()`, even when the customer candidate bytes are unchanged. Customer modules are never reused in fresh trusted state. The stable state name derives from immutable deployment identity, not candidate contents, so release promotion and rollback do not replace platform state. +### Attest the active route + +Active-route attestation proves which provider artifact receives traffic and which fleet specification digest that artifact carries. It starts from provider routing state, not the desired candidate returned by `inspect()`. Every package-owned promotion path attests after promotion: initial provision, every `migrateFleet()` branch, and `rollbackExternalRelease()` all fail closed when the route is absent, ambiguous, or mismatched. The two-version staging window is refused if observed, but the package's own promote paths never observe it because each attestation runs after promotion converges. + +An ordinary Worker attestation performs two provider reads per attempt: read the deployment traffic split, require exactly one version at 100 percent, then read that version's specification-digest binding. A deployment containing two versions is refused even when one has 100 percent and the other has 0 percent; Fleet control never selects a version by highest share. `physicalScriptName` equals `spec.scriptName` by construction because promotion already proves custom-domain ownership for that script. + +A Workers for Platforms attestation performs three provider reads per attempt: read the hostname mapping, resolve the routed physical script, then inspect that script and its digest. The shared convergence helper retries provider propagation for up to 60 seconds by default. Backend constructors accept `clock?` so `observedAt` uses the host's clock. + +Use `attestFleetRecordActiveRoute()` for a single host-side drift read. Cache its result and never invoke it for every status request: one record costs two provider reads on the plain backend or three on Workers for Platforms. Use `attestConvergedActiveRoute()` only around a promote path that must wait for provider convergence. + +### Settle a promotion under the lease + +Pass `settlementFor?` to `migrateFleet()` or `settlement?` to `rollbackExternalRelease()` to run a host callback after the route attests and matches, but before Fleet control writes the settling state. Each callback receives `FleetSettlementContext` with a stable `settlementKey`. Delivery is at least once: a crash after `settle()` succeeds but before the state write repeats the callback under the same key. The host must deduplicate every external effect on `settlementKey`. + +`alreadySettled` is an observational hint, not a deduplication guarantee. It can be false again after a successful callback whose following state write failed. `FleetRecord.settledSettlementKey` lets a later routine convergence skip a settlement whose key was durably recorded, while route attestation still runs. + +The package imposes no callback timeout. By default, the state store renews the lease every five minutes against a 15-minute lifetime for as long as `settle()` runs. A hung callback that keeps renewing therefore holds the deployment lease indefinitely. Keep the callback brief and enqueue slow work. If the process dies, the lease expires 15 minutes after its last successful renewal by default. A throw leaves the migration re-enterable; the next call re-attests and retries the same settlement key. + +The `entry` field distinguishes `migration`, `platform-only`, `ready-convergence`, and `rollback`. For the first three, `prior` is the replaced release when the backend still retains it. For `rollback`, `prior` is the abandoned active release. `prior` is optional-normal, and settlement logic may depend only on the target identity. + +Initial provisioning consults no settlement host. It returns to the caller after route attestation, and the host can settle after return with `attestFleetRecordActiveRoute()`. + +### Upgrade from 0.3 + +The 0.4 public surface has these breaking requirements: + +- `ProvisioningBackend` implementations add required `attestActiveRoute()` +- Public `PlainWorkerRouteApi` implementations add required `inspectActiveWorkerRoute()` +- `ProvisioningBackend.seedDeploymentIdentity()` takes `{ initialExecutionFenceState }` as its fourth argument through `SeedDeploymentIdentityOptions` +- `provisionDeployment()` requires `initialExecutionFenceState`, returns its validation failures asynchronously, accepts `routeAttestation?`, and records the routed artifact version +- `migrateFleet()` accepts `settlementFor?` and `routeAttestation?` +- `rollbackExternalRelease()` accepts `settlement?` and `routeAttestation?` +- `WranglerLoopBackend` and `WorkersForPlatformsBackend` constructors accept `clock?` +- `FleetRecord` adds `settledSettlementKey`; `D1FleetStateStore` adds the nullable text column automatically on first open + +Fleet control 0.4 depends on exactly one matching Flowsafe 0.20 runtime copy. Upgrade a direct Flowsafe 0.19 dependency at the same time to avoid a second nominal runtime copy. + ## Switch a plain deployment to Workers for Platforms Use `switchPlainDeploymentToWorkersForPlatforms()` only for an existing platform-authored deployment that must accept external releases without moving D1 data or Durable Object namespaces. The switch stores its intent in the canonical fleet row and holds the same `FleetStateLease` used by provision, migration, rollback, and decommission. Those lifecycle operations reject an active switch. diff --git a/docs/security-threat-model.md b/docs/security-threat-model.md index ac19b2e..8b79a37 100644 --- a/docs/security-threat-model.md +++ b/docs/security-threat-model.md @@ -141,6 +141,8 @@ Run termination keeps the same boundary. The Worker sends the trusted execution External maintenance is a separate, narrower channel. The candidate may relay a fleet-private Ed25519 capability, but it cannot mint one or read `MAINTENANCE_ADMIN_SECRET`. The global dispatcher verifies the public signature and the bound operation, tenant, environment, physical script, specification digest, expiry, and nonce before it invokes customer code. The trusted maintenance object verifies the same capability against its static tenant and environment, rejects deployment-identity authorization for maintenance routes in this mode, consumes mutation nonces atomically, and signs the exact result with its per-state HMAC secret. Fleet control accepts only that signed result. Status verification is read-only, and maintenance mutation is bounded by a request timeout shorter than the active lease. +The execution-fence and inventory admin routes use a stricter `MAINTENANCE_ADMIN_SECRET` boundary than the maintenance routes: they return `503` when the secret is absent and never delegate authentication to fleet capabilities. The fence can halt every execution entry in the deployment, so the host owns and audits transition policy. The deployment-identity gate still runs first; a mismatched binding or sentinel cannot use fence administration to reach an unverified database. + A missing binding, invalid sentinel schema, missing or extra owner row, malformed tag, caller-credential mismatch, or tag mismatch fails closed. The Worker returns `503`; Durable Object initialization refuses the request. ### Opaque server-minted ids diff --git a/packages/agent-starter/CHANGELOG.md b/packages/agent-starter/CHANGELOG.md index 3daeef8..f585fa5 100644 --- a/packages/agent-starter/CHANGELOG.md +++ b/packages/agent-starter/CHANGELOG.md @@ -1,5 +1,12 @@ # anchorage-agent-starter +## 0.0.16 + +### Patch Changes + +- Updated dependencies [1212ba5] + - @proofoftech/flowsafe@0.20.0 + ## 0.0.15 ### Patch Changes diff --git a/packages/agent-starter/package.json b/packages/agent-starter/package.json index 4ce5470..3922dd3 100644 --- a/packages/agent-starter/package.json +++ b/packages/agent-starter/package.json @@ -1,6 +1,6 @@ { "name": "anchorage-agent-starter", - "version": "0.0.15", + "version": "0.0.16", "private": true, "description": "Production-shaped Cloudflare Workers starter for durable, approval-gated Anchorage agents", "type": "module", diff --git a/packages/agent-starter/src/config.ts b/packages/agent-starter/src/config.ts index 04bb147..97ffb75 100644 --- a/packages/agent-starter/src/config.ts +++ b/packages/agent-starter/src/config.ts @@ -33,6 +33,16 @@ let githubOwnershipMemo: } | undefined; +/** + * The host's audit sink: one structured line to Workers Logs. Shared by every + * router and by the maintenance tick so a deployment's audit trail is one + * stream, and it lives here rather than in the Worker entry because workerd + * rejects a non-handler export from an entry module. + */ +export function audit(event: unknown): void { + console.log(JSON.stringify(event)); +} + export function buildVerifier(env: Env): TokenVerifier { const key = [ env.AUTH_HMAC_SECRET ?? '', diff --git a/packages/agent-starter/src/conformance/state-durable-objects.ts b/packages/agent-starter/src/conformance/state-durable-objects.ts index 4384d38..19855c9 100644 --- a/packages/agent-starter/src/conformance/state-durable-objects.ts +++ b/packages/agent-starter/src/conformance/state-durable-objects.ts @@ -10,6 +10,7 @@ import { type DurableObjectRunLifecycleHooks, DurableObjectRunner, doErrorResponse, + executionFenceFor, verifyDurableObjectDeploymentRequest, } from '@proofoftech/flowsafe/do-runner'; import { @@ -206,6 +207,11 @@ export class ConformanceState { deploymentTag: this.#env.DEPLOYMENT_TENANT, resumeRun: (record, decision) => this.#topology().resumeRecord(record, decision), + // The package's per-binding memo, so this artifact gates the SAME + // database as the runs it decides on: `decide()` commits before it + // resumes, and a conformance state script on a locked deployment must + // refuse rather than record a decision with nothing behind it. + executionFence: executionFenceFor(this.#env.DB), }, ); } diff --git a/packages/agent-starter/src/durable-objects.ts b/packages/agent-starter/src/durable-objects.ts index 04a913e..f9847e4 100644 --- a/packages/agent-starter/src/durable-objects.ts +++ b/packages/agent-starter/src/durable-objects.ts @@ -70,7 +70,9 @@ import { modelConfig, SYSTEM_PRINCIPAL_ID } from './config.js'; import { starterRunnerLifecycleConfig } from './principal-context.js'; import { createComposedStorage, + executionFence, schedulesStore, + startIdempotency, subscriptionStoreFactory, } from './storage.js'; import { defineWorkflows } from './workflows.js'; @@ -152,6 +154,7 @@ export class StarterSignalProviderHost extends SignalProviderHost { env.DEPLOYMENT_IDENTITY_SECRET, ), providers: [github], + executionFence: executionFence(env.DB), }; } } @@ -193,6 +196,10 @@ export class StarterThread extends ThreadDurableObject { approvalService: () => { this.#approvalService ??= new ApprovalService({ store: approvals, + // decide() COMMITS and only then resumes, so the fence belongs at + // the service: a decision recorded against a locked deployment + // would be durable with nothing behind it. + executionFence: executionFence(env.DB), ...(this.env.STREAM_TICKET_SECRET ? { stream: (event) => @@ -216,6 +223,14 @@ export class StarterThread extends ThreadDurableObject { requestContextForRun: agentHost.requestContextForRun( approvalGrantProvider(approvals), ), + // The composed store hides the binding init would have fenced from, so + // this thread DO names it: the deployment execution fence must live in + // the SAME database as the state it fences. + executionFence: executionFence(env.DB), + // Same binding, same reasoning: the agent topology reserves keyed + // agent starts into this store, and this runtime is what settles them + // when the run it dispatched reaches terminal. + startIdempotency: startIdempotency(env.DB), }, ); this.#threadInit = threadInit; @@ -400,6 +415,7 @@ export class StarterBackgroundTasks { }, }, execution: true, + executionFence: executionFence(this.env.DB), manager: { globalConcurrency: 10, perAgentConcurrency: 3, @@ -445,7 +461,10 @@ export class StarterBackgroundTasks { const resources = approvalStoreFactoryFor(this.env.DB).resources(); const host = await this.#ensureHost(); const route = createBackgroundTaskRoutes({ - manager: host.manager, + // The HOST, not its manager: the manager also carries enqueue and the + // executor registry, and a route must not be one property access + // away from putting an unfenced task body on this deployment. + manager: host, authorize: async (scope) => { const checks: Promise[] = []; if (scope.runId !== undefined) { diff --git a/packages/agent-starter/src/maintenance.ts b/packages/agent-starter/src/maintenance.ts new file mode 100644 index 0000000..8d3f465 --- /dev/null +++ b/packages/agent-starter/src/maintenance.ts @@ -0,0 +1,255 @@ +// SPDX-License-Identifier: Apache-2.0 +// The host's unattended maintenance duty, in a module of its own rather than in +// the Worker entry: workerd rejects a non-handler export from an entry module, +// so anything a test must reach has to live beside it (the same shape +// flowsafe's own deploy/crons.ts uses). +// +// What is worth testing here is the WIRING, not the closures: every surface in +// this host takes its execution fence from `executionFence(env.DB)`, and the +// one that must not be missed is this tick — an unfenced tick claims a due fire +// through the schedules CAS (which advances `nextFireAt`) and the fenced +// runtime then refuses the start, so the fire is consumed and never runs. + +import { createAgentThreadTopology } from '@proofoftech/flowsafe/agent-host'; +import { + createDoRunTopology, + createThreadTopology, + queueApprovalForSuspension, + RunRouteError, +} from '@proofoftech/flowsafe/host-kit'; +import { + createScheduleTargetPolicy, + createScheduleTick, + parseScheduleAgentDispatchReceipt, +} from '@proofoftech/flowsafe/schedules'; +import { createNotificationDispatchTick } from '@proofoftech/flowsafe/signals'; + +import { STARTER_AGENT_META } from './agent.js'; +import { audit, SYSTEM_PRINCIPAL_ID } from './config.js'; +import { contextForResourceOwner, systemContext } from './principal-context.js'; +import { + executionFence, + notificationsStore, + schedulesStore, + startIdempotency, +} from './storage.js'; +import { WORKFLOWS } from './workflows.js'; + +/** The catalog every schedule target is rechecked against, at create AND at fire. */ +export const scheduleTargetPolicy = createScheduleTargetPolicy({ + workflows: WORKFLOWS, + agents: [STARTER_AGENT_META], +}); + +/** + * The deployment's cron duty: claim and fire due schedules, then dispatch due + * notifications. Both passes read the SAME fence store as the runtime and the + * routers, so a fenced deployment neither claims a fire nor burns a + * notification's delivery attempts. + */ +export function starterMaintenanceTick(env: Env): () => Promise { + // ONE store, named once: the file header's claim that both passes gate the + // same fence as the runtime is a claim about identity, and two calls make a + // reader check the memo to believe it. + const fence = executionFence(env.DB); + const runTopology = createDoRunTopology( + env.RUNNER, + env.DEPLOYMENT_IDENTITY_SECRET, + ); + const threadTopology = createThreadTopology( + env.THREAD, + env.DEPLOYMENT_IDENTITY_SECRET, + ); + const agentTopology = createAgentThreadTopology( + env.THREAD, + env.DEPLOYMENT_IDENTITY_SECRET, + { + startIdempotency: startIdempotency(env.DB), + executionFence: fence, + }, + ); + const schedules = createScheduleTick({ + store: schedulesStore(env.DB), + targetPolicy: scheduleTargetPolicy, + executionFence: fence, + start: async ({ workflowId, runId, inputData, scheduleId, dispatchId }) => { + const context = await contextForResourceOwner( + env, + 'schedule', + scheduleId, + 'schedule.fire', + ); + const summary = await runTopology.start({ + workflowId, + runId, + inputData, + principal: context.principal, + scheduleId, + dispatchId, + }); + if (summary.status === 'suspended') { + try { + await queueApprovalForSuspension( + context.service(), + workflowId, + summary, + context.principal.id, + SYSTEM_PRINCIPAL_ID, + ); + } catch (error) { + console.error( + JSON.stringify({ + type: 'scheduled-approval-filing-error', + workflowId, + runId, + error: error instanceof Error ? error.message : String(error), + }), + ); + } + } + return summary; + }, + deploymentTag: env.DEPLOYMENT_TENANT, + startAgent: async ({ + scheduleId, + dispatchId, + target, + runId, + topologyThreadId, + threaded, + entryPath, + requestContext, + streamRequestContext, + providerOptions, + }) => { + const context = await contextForResourceOwner( + env, + 'schedule', + scheduleId, + 'schedule.fire', + ); + const started = await agentTopology.start(context, { + agentId: target.agentId, + runId, + prompt: target.prompt, + entryPath, + scheduleId, + dispatchId, + threaded, + requestContext, + streamRequestContext, + providerOptions, + ...(threaded + ? { + threadId: target.threadId, + resourceId: target.resourceId, + } + : { topologyThreadId }), + }); + return { runId: started.runId }; + }, + signalAgent: async ({ scheduleId, target, dispatchId, runId }) => { + if (!target.threadId || !target.resourceId) { + throw new Error('threaded schedule signal requires memory ids'); + } + const context = await contextForResourceOwner( + env, + 'schedule', + scheduleId, + 'schedule.fire', + ); + const response = await threadTopology.send( + context, + target.threadId, + '/signal/schedule', + { + method: 'POST', + headers: { 'content-type': 'application/json' }, + body: JSON.stringify({ + scheduleId, + dispatchId, + runId, + }), + }, + ); + if (!response.ok) { + throw new RunRouteError( + response.status, + `agent schedule signal failed with status ${response.status}`, + ); + } + const payload = (await response.json()) as { receipt?: unknown }; + const receipt = parseScheduleAgentDispatchReceipt(payload.receipt); + if (!receipt) throw new Error('agent schedule returned no valid receipt'); + return receipt; + }, + status: async (ref) => { + const context = await contextForResourceOwner( + env, + 'schedule', + ref.scheduleId, + 'schedule.fire', + ); + if (ref.target === 'workflow') { + const summary = await runTopology.dispatchStatus( + ref.workflowId, + ref.runId, + ); + if (summary?.status === 'suspended') { + try { + await queueApprovalForSuspension( + context.service(), + ref.workflowId, + summary, + context.principal.id, + SYSTEM_PRINCIPAL_ID, + ); + } catch (error) { + console.error( + JSON.stringify({ + type: 'scheduled-approval-filing-error', + workflowId: ref.workflowId, + runId: ref.runId, + error: error instanceof Error ? error.message : String(error), + }), + ); + } + } + return summary; + } + if (ref.mode === 'signal') { + const state = await schedulesStore(env.DB).agentScheduleDispatchState( + ref.scheduleId, + ref.dispatchId, + ); + if (state.state === 'settled') { + return { + runId: state.receipt.runId, + dispatchReceipt: state.receipt, + }; + } + if (state.state === 'pending') { + throw new Error('agent schedule dispatch remains pending'); + } + return undefined; + } + return agentTopology.dispatchStatus(context, { + agentId: ref.agentId, + threadId: ref.threadId, + runId: ref.runId, + }); + }, + audit, + }); + const notifications = createNotificationDispatchTick({ + storage: notificationsStore(env.DB), + topology: threadTopology, + resolveContext: () => systemContext(env, 'notification-dispatch'), + limit: 100, + executionFence: fence, + }); + return async () => ({ + schedules: await schedules(), + notifications: await notifications(), + }); +} diff --git a/packages/agent-starter/src/principal-context.ts b/packages/agent-starter/src/principal-context.ts index 1305a90..e8bd06d 100644 --- a/packages/agent-starter/src/principal-context.ts +++ b/packages/agent-starter/src/principal-context.ts @@ -20,6 +20,7 @@ import { import { STARTER_AGENT_META } from './agent.js'; import { SYSTEM_PRINCIPAL_ID } from './config.js'; +import { executionFence, startIdempotency } from './storage.js'; export const starterRunnerLifecycleConfig = { systemPrincipalId: SYSTEM_PRINCIPAL_ID, @@ -30,6 +31,10 @@ export const starterRunnerLifecycleConfig = { topology: createAgentThreadTopology( env.THREAD, env.DEPLOYMENT_IDENTITY_SECRET, + { + startIdempotency: startIdempotency(env.DB), + executionFence: executionFence(env.DB), + }, ), contextForPrincipal: (principal, record) => { const target = record.resumeTarget; @@ -70,7 +75,14 @@ export function contextForPrincipal( principal, storeFactory: factory, deploymentTag: env.DEPLOYMENT_TENANT, - buildService: (store) => new ApprovalService({ store }), + buildService: (store) => + new ApprovalService({ + // Same database, same fence as every other surface here: this service + // files and decides approvals for scheduled and system-driven runs, and + // decide() COMMITS before it resumes. + store, + executionFence: executionFence(env.DB), + }), }); } diff --git a/packages/agent-starter/src/storage.ts b/packages/agent-starter/src/storage.ts index fe2665d..6af24f1 100644 --- a/packages/agent-starter/src/storage.ts +++ b/packages/agent-starter/src/storage.ts @@ -1,6 +1,12 @@ // SPDX-License-Identifier: Apache-2.0 -import { createD1Storage } from '@proofoftech/flowsafe/do-runner'; +import { + createD1Storage, + type ExecutionFenceStore, + executionFenceFor, + type StartIdempotencyStore, + startIdempotencyFor, +} from '@proofoftech/flowsafe/do-runner'; import { createScheduleStorageDomains, D1SchedulesStorage, @@ -80,3 +86,35 @@ export function schedulesStore(db: Env['DB']): D1SchedulesStorage { } return store; } + +/** + * The deployment execution fence, one store per D1 binding. + * + * Every fenced surface in this host takes its fence from HERE rather than + * building one: the schedule tick that must not claim a due fire, the routers + * that refuse to author new work, the background-task host, the provider + * poller, and the approval services that decide-then-resume all have to be + * gating the SAME database as the runtime they sit in front of. + * + * The memo is the package's own `executionFenceFor` rather than a fifth copy + * beside the four above — it is keyed on the binding for exactly the reason the + * rest of this file is, and sharing it means this host and the flowsafe + * internals a route reaches through hand back the same store for one database. + * The local name stays because it is what every call site in this host reads. + */ +export function executionFence(db: Env['DB']): ExecutionFenceStore { + return executionFenceFor(db); +} + +/** + * The deployment's start reservations, one store per D1 binding. + * + * Same rule, same reason, as the fence above: the run router and every agent + * topology in this host RESERVE against a key, and the runtimes inside the run + * and thread objects SETTLE those same rows when a run ends. Two stores over + * two bindings would be two tables answering the same key, and the settle would + * land nowhere. + */ +export function startIdempotency(db: Env['DB']): StartIdempotencyStore { + return startIdempotencyFor(db); +} diff --git a/packages/agent-starter/src/worker.ts b/packages/agent-starter/src/worker.ts index 0e68440..83811d7 100644 --- a/packages/agent-starter/src/worker.ts +++ b/packages/agent-starter/src/worker.ts @@ -14,20 +14,13 @@ import { } from '@proofoftech/flowsafe/do-runner'; import { createObjectiveRouter } from '@proofoftech/flowsafe/goals'; import { - createDoRunTopology, createFlowsafeMaintenanceDurableObject, createFlowsafeWorker, createThreadTopology, type FlowsafeWorkerConfig, - queueApprovalForSuspension, RunRouteError, } from '@proofoftech/flowsafe/host-kit'; -import { - createScheduleRouter, - createScheduleTargetPolicy, - createScheduleTick, - parseScheduleAgentDispatchReceipt, -} from '@proofoftech/flowsafe/schedules'; +import { createScheduleRouter } from '@proofoftech/flowsafe/schedules'; import { createSignalProviderHostTopology, createSubscriptionRouter, @@ -36,15 +29,14 @@ import { } from '@proofoftech/flowsafe/signal-providers'; import { createInMemorySignalRateLimiter, - createNotificationDispatchTick, createSignalRouter, } from '@proofoftech/flowsafe/signals'; import { STARTER_AGENT_META } from './agent.js'; import { + audit, buildVerifier, githubResourceAllowed, - SYSTEM_PRINCIPAL_ID, signalAttributeAllowlist, } from './config.js'; import { @@ -55,14 +47,12 @@ import { StarterSignalProviderHost, StarterThread, } from './durable-objects.js'; +import { scheduleTargetPolicy, starterMaintenanceTick } from './maintenance.js'; +import { starterRunnerLifecycleConfig } from './principal-context.js'; import { - contextForResourceOwner, - starterRunnerLifecycleConfig, - systemContext, -} from './principal-context.js'; -import { - notificationsStore, + executionFence, schedulesStore, + startIdempotency, subscriptionStoreFactory, threadStateStore, } from './storage.js'; @@ -77,10 +67,6 @@ export { }; const github = githubSignalProvider(); -const scheduleTargetPolicy = createScheduleTargetPolicy({ - workflows: WORKFLOWS, - agents: [STARTER_AGENT_META], -}); const signalRateLimit = createInMemorySignalRateLimiter({ limit: 120, windowMs: 60_000, @@ -94,10 +80,6 @@ function json(payload: unknown, status = 200): Response { return Response.json(payload, { status }); } -function audit(event: unknown): void { - console.log(JSON.stringify(event)); -} - function webhookRouter(env: Env) { let router = webhookRouters.get(env.DB); if (!router) { @@ -112,6 +94,7 @@ function webhookRouter(env: Env) { secretForProvider: (providerId) => providerId === 'github' ? env.GITHUB_WEBHOOK_SECRET : undefined, audit, + executionFence: executionFence(env.DB), }); webhookRouters.set(env.DB, router); } @@ -173,6 +156,10 @@ const workerConfig = { validateThreadTarget: createAgentThreadTopology( env.THREAD, env.DEPLOYMENT_IDENTITY_SECRET, + { + startIdempotency: startIdempotency(env.DB), + executionFence: executionFence(env.DB), + }, ).requireBoundThread, knownProviders: ['github'], authorizeMutation: ({ @@ -214,6 +201,10 @@ const workerConfig = { topology: createAgentThreadTopology( env.THREAD, env.DEPLOYMENT_IDENTITY_SECRET, + { + startIdempotency: startIdempotency(env.DB), + executionFence: executionFence(env.DB), + }, ), }), buildSignalRouter: (resolve, env) => @@ -234,9 +225,14 @@ const workerConfig = { validateThreadTarget: createAgentThreadTopology( env.THREAD, env.DEPLOYMENT_IDENTITY_SECRET, + { + startIdempotency: startIdempotency(env.DB), + executionFence: executionFence(env.DB), + }, ).requireBoundThread, maxRunsCap: 50, audit, + executionFence: executionFence(env.DB), }), buildScheduleRouter: (resolve, env) => createScheduleRouter({ @@ -246,214 +242,17 @@ const workerConfig = { validateThreadTarget: createAgentThreadTopology( env.THREAD, env.DEPLOYMENT_IDENTITY_SECRET, + { + startIdempotency: startIdempotency(env.DB), + executionFence: executionFence(env.DB), + }, ).requireBoundThread, maxSchedules: 100, minFireIntervalMs: 60_000, audit, + executionFence: executionFence(env.DB), }), - scheduleTick: (env) => { - const runTopology = createDoRunTopology( - env.RUNNER, - env.DEPLOYMENT_IDENTITY_SECRET, - ); - const threadTopology = createThreadTopology( - env.THREAD, - env.DEPLOYMENT_IDENTITY_SECRET, - ); - const agentTopology = createAgentThreadTopology( - env.THREAD, - env.DEPLOYMENT_IDENTITY_SECRET, - ); - const schedules = createScheduleTick({ - store: schedulesStore(env.DB), - targetPolicy: scheduleTargetPolicy, - start: async ({ - workflowId, - runId, - inputData, - scheduleId, - dispatchId, - }) => { - const context = await contextForResourceOwner( - env, - 'schedule', - scheduleId, - 'schedule.fire', - ); - const summary = await runTopology.start({ - workflowId, - runId, - inputData, - principal: context.principal, - scheduleId, - dispatchId, - }); - if (summary.status === 'suspended') { - try { - await queueApprovalForSuspension( - context.service(), - workflowId, - summary, - context.principal.id, - SYSTEM_PRINCIPAL_ID, - ); - } catch (error) { - console.error( - JSON.stringify({ - type: 'scheduled-approval-filing-error', - workflowId, - runId, - error: error instanceof Error ? error.message : String(error), - }), - ); - } - } - return summary; - }, - deploymentTag: env.DEPLOYMENT_TENANT, - startAgent: async ({ - scheduleId, - dispatchId, - target, - runId, - topologyThreadId, - threaded, - entryPath, - requestContext, - streamRequestContext, - providerOptions, - }) => { - const context = await contextForResourceOwner( - env, - 'schedule', - scheduleId, - 'schedule.fire', - ); - const started = await agentTopology.start(context, { - agentId: target.agentId, - runId, - prompt: target.prompt, - entryPath, - scheduleId, - dispatchId, - threaded, - requestContext, - streamRequestContext, - providerOptions, - ...(threaded - ? { - threadId: target.threadId, - resourceId: target.resourceId, - } - : { topologyThreadId }), - }); - return { runId: started.runId }; - }, - signalAgent: async ({ scheduleId, target, dispatchId, runId }) => { - if (!target.threadId || !target.resourceId) { - throw new Error('threaded schedule signal requires memory ids'); - } - const context = await contextForResourceOwner( - env, - 'schedule', - scheduleId, - 'schedule.fire', - ); - const response = await threadTopology.send( - context, - target.threadId, - '/signal/schedule', - { - method: 'POST', - headers: { 'content-type': 'application/json' }, - body: JSON.stringify({ - scheduleId, - dispatchId, - runId, - }), - }, - ); - if (!response.ok) { - throw new RunRouteError( - response.status, - `agent schedule signal failed with status ${response.status}`, - ); - } - const payload = (await response.json()) as { receipt?: unknown }; - const receipt = parseScheduleAgentDispatchReceipt(payload.receipt); - if (!receipt) - throw new Error('agent schedule returned no valid receipt'); - return receipt; - }, - status: async (ref) => { - const context = await contextForResourceOwner( - env, - 'schedule', - ref.scheduleId, - 'schedule.fire', - ); - if (ref.target === 'workflow') { - const summary = await runTopology.dispatchStatus( - ref.workflowId, - ref.runId, - ); - if (summary?.status === 'suspended') { - try { - await queueApprovalForSuspension( - context.service(), - ref.workflowId, - summary, - context.principal.id, - SYSTEM_PRINCIPAL_ID, - ); - } catch (error) { - console.error( - JSON.stringify({ - type: 'scheduled-approval-filing-error', - workflowId: ref.workflowId, - runId: ref.runId, - error: error instanceof Error ? error.message : String(error), - }), - ); - } - } - return summary; - } - if (ref.mode === 'signal') { - const state = await schedulesStore(env.DB).agentScheduleDispatchState( - ref.scheduleId, - ref.dispatchId, - ); - if (state.state === 'settled') { - return { - runId: state.receipt.runId, - dispatchReceipt: state.receipt, - }; - } - if (state.state === 'pending') { - throw new Error('agent schedule dispatch remains pending'); - } - return undefined; - } - return agentTopology.dispatchStatus(context, { - agentId: ref.agentId, - threadId: ref.threadId, - runId: ref.runId, - }); - }, - audit, - }); - const notifications = createNotificationDispatchTick({ - storage: notificationsStore(env.DB), - topology: threadTopology, - resolveContext: () => systemContext(env, 'notification-dispatch'), - limit: 100, - }); - return async () => ({ - schedules: await schedules(), - notifications: await notifications(), - }); - }, + scheduleTick: starterMaintenanceTick, backgroundTasks: { completedTtlMs: 60 * 60 * 1_000, failedTtlMs: 24 * 60 * 60 * 1_000, diff --git a/packages/agent-starter/src/workflows.ts b/packages/agent-starter/src/workflows.ts index 06b7088..e159bac 100644 --- a/packages/agent-starter/src/workflows.ts +++ b/packages/agent-starter/src/workflows.ts @@ -9,7 +9,11 @@ import { } from '@proofoftech/flowsafe/host-kit'; import { z } from 'zod'; -import { createComposedStorage } from './storage.js'; +import { + createComposedStorage, + executionFence, + startIdempotency, +} from './storage.js'; export const WORKFLOWS: ReadonlyArray = [ { @@ -28,6 +32,16 @@ export function defineWorkflows(env: Env): RunnerRuntime { requestContextForRun: approvalGrantProvider( approvalStoreFactoryFor(env.DB).store(), ), + // The composed store hides the binding init would have fenced from, so + // the run object names it: the deployment execution fence must live in + // the SAME database as the state it fences. + executionFence: executionFence(env.DB), + // The same binding again, and not optional in practice: the run router + // reserves keyed starts into this store, and THIS runtime is the layer + // that sees a run reach terminal — so it is the only one that can mark + // the reservation spent. `DurableObjectRunner.build()` refuses to serve + // a database-backed runtime without it. + startIdempotency: startIdempotency(env.DB), }, ); const schema = z.object({ message: z.string().min(1).max(5_000) }); diff --git a/packages/agent-starter/test/execution-fence-composition.test.ts b/packages/agent-starter/test/execution-fence-composition.test.ts new file mode 100644 index 0000000..98c23b3 --- /dev/null +++ b/packages/agent-starter/test/execution-fence-composition.test.ts @@ -0,0 +1,172 @@ +// SPDX-License-Identifier: Apache-2.0 +// The composition proof the unit tests cannot make. flowsafe pins what a fenced +// tick DOES; this pins that THIS HOST wired one — the failure the required +// `executionFence` option exists to prevent is a deployment where the runtime is +// fenced and the tick is not, and every flowsafe test would still pass. +// +// The loss is silent and total: an unfenced tick claims a due fire through the +// schedules CAS, which advances `nextFireAt`, and the fenced runtime then +// refuses the start. The fire is consumed and never runs, and nothing reports +// it — which is why the assertion below is on the schedule ROW, not on the +// tally the pass returned. + +import { ExecutionFenceStore } from '@proofoftech/flowsafe/do-runner'; +import { describe, expect, it } from 'vitest'; + +import { starterMaintenanceTick } from '../src/maintenance.js'; +import { schedulesStore } from '../src/storage.js'; + +interface SqliteStatement { + get(...params: unknown[]): unknown; + run(...params: unknown[]): unknown; + all(...params: unknown[]): unknown[]; +} + +interface SqliteDatabase { + prepare(sql: string): SqliteStatement; + exec(sql: string): void; +} + +function openSqlite(): SqliteDatabase { + const getBuiltin = ( + globalThis as { + process?: { getBuiltinModule?: (id: string) => unknown }; + } + ).process?.getBuiltinModule; + if (!getBuiltin) { + throw new Error('node:sqlite unavailable; tests require Node.js 22.13+'); + } + const mod = getBuiltin('node:sqlite') as { + DatabaseSync: new (path: string) => SqliteDatabase; + }; + return new mod.DatabaseSync(':memory:'); +} + +function sqliteUnitDatabase(db: SqliteDatabase): unknown { + const runSync = Symbol('runSync'); + + function statement(sql: string, params: unknown[]): Record { + const execute = () => { + const outcome = db.prepare(sql).run(...params) as { + changes?: number | bigint; + }; + return { + success: true, + meta: { changes: Number(outcome?.changes ?? 0) }, + }; + }; + return { + bind: (...values: unknown[]) => statement(sql, values), + first: async (column?: string) => { + const row = db.prepare(sql).get(...params) as + | Record + | undefined; + if (row === undefined) return null; + return column === undefined ? row : (row[column] ?? null); + }, + run: async () => execute(), + [runSync]: execute, + all: async () => ({ + success: true, + results: db.prepare(sql).all(...params), + meta: {}, + }), + }; + } + + return { + prepare: (sql: string) => statement(sql, []), + batch: async ( + statements: Array<{ + run: () => Promise; + [runSync]?: () => unknown; + }>, + ) => { + db.exec('BEGIN IMMEDIATE'); + try { + const results = []; + for (const prepared of statements) { + results.push( + prepared[runSync] ? prepared[runSync]() : await prepared.run(), + ); + } + db.exec('COMMIT'); + return results; + } catch (error) { + db.exec('ROLLBACK'); + throw error; + } + }, + }; +} + +const NOW = 1_750_000_000_000; + +/** + * A namespace stub. The tick builds its run/thread topologies eagerly, but a + * fenced pass never addresses one — reaching this is the failure. + */ +function namespace(): Env['RUNNER'] { + const unreachable = () => { + throw new Error( + 'a fenced maintenance pass addressed a Durable Object — it claimed work', + ); + }; + return { + idFromName: unreachable, + idFromString: unreachable, + newUniqueId: unreachable, + get: unreachable, + } as unknown as Env['RUNNER']; +} + +function starterEnv(db: Env['DB']): Env { + return { + DB: db, + DEPLOYMENT_TENANT: 'acme', + DEPLOYMENT_IDENTITY_SECRET: 'test-deployment-identity-secret-0001', + RUNNER: namespace(), + THREAD: namespace(), + } as unknown as Env; +} + +describe('starter maintenance tick and the deployment execution fence', () => { + it('leaves a due schedule row untouched while the deployment is migration-locked', async () => { + // #given — this host's own tick over a database whose fence is locked. The + // fence store is built from the SAME binding the tick's own + // `executionFence(env.DB)` resolves, which is the wiring under test: a tick + // pointed at another database would read `open` here and claim. + const db = sqliteUnitDatabase(openSqlite()) as Env['DB']; + const env = starterEnv(db); + const fence = new ExecutionFenceStore(db); + await fence.seed('migration-locked'); + + const store = schedulesStore(db); + const due = { + id: 'schedule_fenced', + target: { + type: 'workflow' as const, + workflowId: 'starter-echo', + inputData: { topic: 'x' }, + }, + cron: '* * * * *', + status: 'active' as const, + nextFireAt: NOW - 1_000, + createdAt: NOW, + updatedAt: NOW, + metadata: {}, + }; + await store.createSchedule(due); + const before = await store.getSchedule(due.id); + + // #when — the cron duty runs, exactly as the maintenance Durable Object + // invokes it. + await starterMaintenanceTick(env)(); + + // #then — the row is byte-identical. Nothing was claimed, so the fire is + // still due and the deployment taking over will run it. + await expect(store.getSchedule(due.id)).resolves.toEqual(before); + await expect(store.listDueSchedules(NOW, 10)).resolves.toHaveLength(1); + await expect(store.listTriggers(due.id)).resolves.toEqual([]); + }); +}); diff --git a/packages/fleet-control/CHANGELOG.md b/packages/fleet-control/CHANGELOG.md index 98481da..4d8dd6e 100644 --- a/packages/fleet-control/CHANGELOG.md +++ b/packages/fleet-control/CHANGELOG.md @@ -1,5 +1,30 @@ # @proofoftech/fleet-control +## 0.4.0 + +### Minor Changes + +- 1212ba5: Fleet Control now attests the release that is actually serving traffic after every package-owned promotion and supports lease-held, idempotent settlement callbacks. + + This changes the public control-plane contract: + + - **BREAKING:** `ProvisioningBackend.attestActiveRoute()` is required. Public `PlainWorkerRouteApi` implementations must also provide `inspectActiveWorkerRoute()`. + - **BREAKING:** the fourth argument to `ProvisioningBackend.seedDeploymentIdentity()` is now `SeedDeploymentIdentityOptions`, shaped as `{ initialExecutionFenceState }`. Provisioning exports now include `InitialExecutionFenceState`. + - **BREAKING:** `provisionDeployment()` requires `initialExecutionFenceState`, is asynchronous even when entry validation fails, and accepts `routeAttestation?`. Its ready record uses the attested routed `artifactVersion`, which can differ from the inspected candidate value recorded by earlier releases. + - **BEHAVIOR CHANGE:** `provisionDeployment()`, every `migrateFleet()` promotion branch, and `rollbackExternalRelease()` attest after promotion and fail closed when routing is absent, split across two versions, or does not match the expected release. Earlier releases could complete from desired-state inspection alone. + - Active-route exports now include `attestFleetRecordActiveRoute`, `attestConvergedActiveRoute`, `ActiveRouteAttestation`, `ActiveRouteAttestationError`, `ActiveRouteExpectation`, `AttestConvergedActiveRouteOptions`, and `ObservedActiveRoute`. + - Settlement exports now include `fleetSettlementKey`, `FleetSettlementContext`, `FleetSettlementEntry`, and `FleetSettlementHost`. `migrateFleet()` accepts `settlementFor?` and `routeAttestation?`; `rollbackExternalRelease()` accepts `settlement?` and `routeAttestation?`. + - Settlement callbacks run at least once under the deployment lease and are keyed by `settlementKey`. Hosts must deduplicate external effects by that key. The package records `FleetRecord.settledSettlementKey` so routine convergence skips an already recorded settlement while still attesting the route. Existing fleet databases add the nullable column automatically on first open. + - Both backend constructors accept `clock?` for attestation timestamps. + - The runtime dependency on `@proofoftech/flowsafe` is published as the exact matching release. A host pinned to an older Flowsafe must upgrade or its package manager can install a second copy, which the deployment boundary does not support. + + Implement the new backend methods before upgrading. Treat route ambiguity as a refusal, cache host-facing attestations instead of reading them per request, pass an explicit initial execution-fence state to provisioning and seeding, and make every settlement effect idempotent on `settlementKey`. + +### Patch Changes + +- Updated dependencies [1212ba5] + - @proofoftech/flowsafe@0.20.0 + ## 0.3.4 ### Patch Changes diff --git a/packages/fleet-control/README.md b/packages/fleet-control/README.md index 34086ec..507f8e7 100644 --- a/packages/fleet-control/README.md +++ b/packages/fleet-control/README.md @@ -99,6 +99,16 @@ Workers Routes Write across every zone in the selected account. Fleet control discovers that complete account-filtered zone set and rejects partial token scope instead of accepting a configured zone list. +### Attest and settle promoted releases + +Fleet Control attests provider routing after every package-owned promotion, including provision, every migration branch, and rollback. An ordinary Worker must have exactly one version at 100 percent; a second version at 0 percent still makes the deployment ambiguous and is refused. Workers for Platforms attestation begins with the hostname mapping and inspects the physical script it actually names. The package never chooses a routed version by highest share. + +Use `attestFleetRecordActiveRoute()` for host-side drift reads and cache its result. One call costs two provider reads for a plain Worker or three for Workers for Platforms. Use `attestConvergedActiveRoute()` around custom promote paths that must wait for provider propagation. + +`migrateFleet({ settlementFor, routeAttestation })` and `rollbackExternalRelease({ settlement, routeAttestation })` can run a host settlement callback under the deployment lease after attestation. Settlement is delivered at least once under a stable `settlementKey`; deduplicate every external effect by that key. The package applies no callback timeout and renews the lease while `settle()` runs, so enqueue slow work. `prior` identifies the replaced release for migration entries and the abandoned active release for rollback. + +Initial `provisionDeployment()` calls require `initialExecutionFenceState: 'open' | 'migration-locked'`, attest the routed release before committing `ready`, and consult no settlement host. Settle after it returns by calling `attestFleetRecordActiveRoute()`. Backend implementations must add `attestActiveRoute()`, and custom `PlainWorkerRouteApi` implementations must add `inspectActiveWorkerRoute()`. `seedDeploymentIdentity()` now takes `{ initialExecutionFenceState }` as its fourth argument. + ## Import it only from a trusted control plane Account credentials, routing ownership, billing policy, and tenant lifecycle belong to the hosted control plane. Do not import this package into a data-plane Worker, and do not give a Worker that serves tenant requests a Cloudflare API token that reaches it. diff --git a/packages/fleet-control/package.json b/packages/fleet-control/package.json index cd3c651..29cfc46 100644 --- a/packages/fleet-control/package.json +++ b/packages/fleet-control/package.json @@ -1,6 +1,6 @@ { "name": "@proofoftech/fleet-control", - "version": "0.3.4", + "version": "0.4.0", "publishConfig": { "access": "public" }, diff --git a/packages/fleet-control/scripts/credentialed-conformance.mjs b/packages/fleet-control/scripts/credentialed-conformance.mjs index 60f8f3e..71a40b1 100644 --- a/packages/fleet-control/scripts/credentialed-conformance.mjs +++ b/packages/fleet-control/scripts/credentialed-conformance.mjs @@ -1122,6 +1122,7 @@ async function assertSecretPreservingStateUpload(deployment, record) { attestationError = error; } const repaired = await provisionDeployment({ + initialExecutionFenceState: 'open', backend, store: deployment.store, spec: deployment.currentSpec, @@ -1269,6 +1270,7 @@ async function provePlainWorkerSecretVersionChurnTeardown() { let conformanceError; try { const provisioned = await provisionDeployment({ + initialExecutionFenceState: 'open', backend: trackedBackend, store: deployment.store, spec, @@ -1339,6 +1341,7 @@ async function provePlainWorkerSecretVersionChurnTeardown() { async function provisionV1(deployment) { deployment.v1Result = await provisionDeployment({ + initialExecutionFenceState: 'open', backend, store: deployment.store, spec: deployment.initialSpec, @@ -1399,10 +1402,15 @@ async function assertTenantIsolation() { let sentinelOwner; await withDeploymentMutationFence(deployments[0], async (fence) => { try { + // The fence state is irrelevant to what this probe asserts (the sentinel + // must refuse a re-stamp), but the protocol requires one; 'open' is the + // state this already-provisioned database is in, so a refusal that + // somehow did not happen could not also silently close its fence. await backend.seedDeploymentIdentity( firstDatabase, deployments[1].initialSpec.tenantTag, fence, + { initialExecutionFenceState: 'open' }, ); } catch (error) { mismatchRejected = /already belongs|refusing to re-stamp/.test( diff --git a/packages/fleet-control/scripts/packed-consumer-test.mjs b/packages/fleet-control/scripts/packed-consumer-test.mjs index 56e3abb..654d7c7 100644 --- a/packages/fleet-control/scripts/packed-consumer-test.mjs +++ b/packages/fleet-control/scripts/packed-consumer-test.mjs @@ -181,6 +181,7 @@ try { await writeFile( join(consumerDirectory, 'consumer.ts'), `import { + ActiveRouteAttestationError, CloudflareProvisioningClient, D1CloudflareApiRateCoordinator, ProcessLocalCloudflareApiRateCoordinator, @@ -188,16 +189,33 @@ try { WorkersForPlatformsBackend, WorkersForPlatformsBackendSwitchProvider, WranglerLoopBackend, + attestConvergedActiveRoute, + attestFleetRecordActiveRoute, auditFleetDrift, decommissionDeployment, forceDecommissionDeployment, deploymentSpecDigest, deriveStateEgressCredential, + fleetSettlementKey, provisionDeployment, validateDeploymentSpec, + type ActiveRouteAttestation, + type ActiveRouteExpectation, + type AttestConvergedActiveRouteOptions, type CloudflareApiRateCoordinator, type DeploymentEgressPolicy, + type DeploymentSpec, + type FleetRecord, + type FleetSettlementContext, + type FleetSettlementEntry, + type FleetSettlementHost, type FleetStateDatabase, + type InitialExecutionFenceState, + type ObservedActiveRoute, + type PlainWorkerCustomDomain, + type PlainWorkerRouteApi, + type ProvisioningBackend, + type SeedDeploymentIdentityOptions, type WorkersForPlatformsApi, } from '@proofoftech/fleet-control'; import type { FleetDispatchEnv } from '@proofoftech/fleet-control/workers/dispatch'; @@ -215,7 +233,69 @@ declare const database: FleetStateDatabase; declare const api: WorkersForPlatformsApi; declare const policy: DeploymentEgressPolicy; declare const coordinator: CloudflareApiRateCoordinator; +declare const deploymentSpec: DeploymentSpec; +declare const provisioningBackend: ProvisioningBackend; +declare const fleetRecord: FleetRecord; +declare const plainWorkerRouteApi: PlainWorkerRouteApi; +// The provisioning-time fence state a control plane has to choose. Named here +// because it is a REQUIRED provisionDeployment option: a consumer that cannot +// import its type cannot type its own provisioning wrapper. +declare const initialExecutionFenceState: InitialExecutionFenceState; +const lockedAtBirth: InitialExecutionFenceState = 'migration-locked'; +// The options object seedDeploymentIdentity takes. A consumer implementing its +// own ProvisioningBackend has to name this type to declare that method, and it +// is where future provisioning context lands without another positional. +const seedOptions: SeedDeploymentIdentityOptions = { + initialExecutionFenceState: lockedAtBirth, +}; +// A consumer implementing its own ProvisioningBackend has to name the +// attestation it returns, and a host reading one has to name what it compares +// against, so both the result and the expectation are part of the surface. +declare const routeAttestation: ActiveRouteAttestation; +const routeExpectation: ActiveRouteExpectation = { + specDigest: routeAttestation.specDigest, + artifactVersion: routeAttestation.artifactVersion, +}; +const routeAttestationOptions: AttestConvergedActiveRouteOptions = { + convergenceBudgetMs: 60_000, +}; +const activeRouteRead: Promise = + provisioningBackend.attestActiveRoute(deploymentSpec); +type SettledSettlementKeyIsOptional = {} extends Pick< + FleetRecord, + 'settledSettlementKey' +> + ? true + : false; +const settledSettlementKeyIsOptional: SettledSettlementKeyIsOptional = true; +const settledSettlementKey: string | undefined = + fleetRecord.settledSettlementKey; +const customDomain: PlainWorkerCustomDomain = { + id: 'domain-id', + hostname: 'acme.example.test', + service: 'acme-production', +}; +const activePlainWorkerRoute = + plainWorkerRouteApi.inspectActiveWorkerRoute(customDomain.service); +// The refusal's payload, which is what a host logs when a route cannot be +// attested; unusable without its type. +declare const observedRoute: ObservedActiveRoute; +// A settling host is written entirely against these types: the callback shape, +// the context it receives, and the entry it must switch on to interpret +// \`prior\`. A consumer that cannot name all three cannot implement one. +const settlementHost: FleetSettlementHost = { + async settle(context: FleetSettlementContext): Promise { + const entry: FleetSettlementEntry = context.entry; + void entry; + void context.settlementKey; + void context.alreadySettled; + void context.attestation.physicalScriptName; + void context.target.specDigest; + void context.prior?.physicalScriptName; + }, +}; +void ActiveRouteAttestationError; void CloudflareProvisioningClient; void D1CloudflareApiRateCoordinator; void ProcessLocalCloudflareApiRateCoordinator; @@ -224,12 +304,15 @@ void WorkersForPlatformsBackend; void WorkersForPlatformsBackendSwitchProvider; void WranglerLoopBackend; void StateEgress; +void attestConvergedActiveRoute; +void attestFleetRecordActiveRoute; void auditFleetDrift; void createEgressProxyFetch; void decommissionDeployment; void forceDecommissionDeployment; void deploymentSpecDigest; void deriveStateEgressCredential; +void fleetSettlementKey; void provisionDeployment; void validateDeploymentSpec; void dispatchEnv; @@ -239,16 +322,37 @@ void database; void api; void policy; void coordinator; +void deploymentSpec; +void provisioningBackend; +void fleetRecord; +void plainWorkerRouteApi; +void initialExecutionFenceState; +void lockedAtBirth; +void seedOptions; +void routeAttestation; +void routeExpectation; +void routeAttestationOptions; +void activeRouteRead; +void settledSettlementKeyIsOptional; +void settledSettlementKey; +void customDomain; +void activePlainWorkerRoute; +void observedRoute; +void settlementHost; `, ); await writeFile( join(consumerDirectory, 'runtime.mjs'), `import assert from 'node:assert/strict'; import { + ActiveRouteAttestationError, ProcessLocalCloudflareApiRateCoordinator, ProvisioningError, WorkersForPlatformsBackend, + attestConvergedActiveRoute, + attestFleetRecordActiveRoute, deploymentSpecDigest, + fleetSettlementKey, } from '@proofoftech/fleet-control'; // Every export entry must load. The three Workers entries are default-export @@ -268,6 +372,11 @@ assert.equal(typeof auditConsumer.default.queue, 'function'); assert.equal(typeof deploymentSpecDigest, 'function'); assert.equal(typeof ProcessLocalCloudflareApiRateCoordinator, 'function'); assert.ok(new ProvisioningError('probe') instanceof Error); +assert.equal(typeof ActiveRouteAttestationError, 'function'); +assert.ok(new ActiveRouteAttestationError('probe', {}) instanceof Error); +assert.equal(typeof attestConvergedActiveRoute, 'function'); +assert.equal(typeof attestFleetRecordActiveRoute, 'function'); +assert.equal(typeof fleetSettlementKey, 'function'); // The trusted-configuration constructor must fail closed. This is the barrier // that makes a published fleet-control inert without control-plane inputs, so diff --git a/packages/fleet-control/src/active-route.ts b/packages/fleet-control/src/active-route.ts new file mode 100644 index 0000000..50ca426 --- /dev/null +++ b/packages/fleet-control/src/active-route.ts @@ -0,0 +1,262 @@ +// SPDX-License-Identifier: Apache-2.0 + +import type { + ActiveRouteAttestation, + DeploymentSpec, + FleetRecord, + ObservedActiveRoute, + ProvisioningBackend, +} from './types.js'; + +/** + * The default 60-second convergence budget stays well inside the default + * five-minute fleet lease renewal interval, so a promote path that attests + * while holding the lease cannot spend its lease waiting on the provider. The + * default schedule caps a single convergence at ten attempts. Each attempt + * spends two provider reads on the plain-Worker backend or three on Workers for + * Platforms — twenty to thirty reads at worst against the default shared + * 1,100-request-per-five-minute window the rate coordinator fences. + */ +const DEFAULT_CONVERGENCE_BUDGET_MS = 60_000; +const DEFAULT_INITIAL_RETRY_DELAY_MS = 1_000; +const DEFAULT_MAX_RETRY_DELAY_MS = 8_000; + +/** + * The artifact version a record carries before its first release settles. It is + * a sentinel, not a version: nothing the provider can report will ever equal + * it, so an expectation carrying it means "no artifact expectation yet" and is + * matched on the specification digest alone. + * + * Exported because settlement resolves the same sentinel when it builds the + * key a host deduplicates on. Two copies of this string that drifted apart + * would let one module waive the version check while the other kept `pending` + * in the key — defeating exactly the deduplication both exist to protect. + */ +export const PENDING_ARTIFACT_VERSION = 'pending'; + +export class ActiveRouteAttestationError extends Error { + /** + * Whatever the provider did report. Always present, including when the + * refusal is "it reported nothing", where it is empty. + */ + readonly observed: ObservedActiveRoute; + /** + * The last complete attestation, present only when the provider answered + * fully and the answer was rejected for disagreeing with the expectation. + * A refusal raised because no attestation could be formed leaves it unset. + */ + readonly attestation: ActiveRouteAttestation | undefined; + + constructor( + message: string, + observed: ObservedActiveRoute, + options: { + readonly cause?: unknown; + readonly attestation?: ActiveRouteAttestation; + } = {}, + ) { + super( + message, + options.cause === undefined ? undefined : { cause: options.cause }, + ); + this.name = 'ActiveRouteAttestationError'; + this.observed = observed; + this.attestation = options.attestation; + } +} + +/** The release identity a caller believes should be serving traffic. */ +export interface ActiveRouteExpectation { + readonly specDigest: string; + /** + * The provider artifact expected to be routed, or `'pending'` when the caller + * has no artifact expectation yet and the digest alone decides the match. + */ + readonly artifactVersion: string; +} + +export interface AttestConvergedActiveRouteOptions { + readonly clock?: () => number; + /** Injected so a test drives the backoff without spending wall-clock time. */ + readonly sleep?: (delayMs: number) => Promise; + readonly convergenceBudgetMs?: number; + readonly initialRetryDelayMs?: number; + readonly maxRetryDelayMs?: number; +} + +function positiveInteger(value: number, name: string): number { + if (!Number.isSafeInteger(value) || value < 1) { + throw new Error(`${name} must be a positive integer`); + } + return value; +} + +function attestationMatches( + attestation: ActiveRouteAttestation, + expected: ActiveRouteExpectation, +): boolean { + return ( + attestation.specDigest === expected.specDigest && + (expected.artifactVersion === PENDING_ARTIFACT_VERSION || + attestation.artifactVersion === expected.artifactVersion) + ); +} + +function observedFrom( + attestation: ActiveRouteAttestation, +): ObservedActiveRoute { + return { + routedScriptName: attestation.physicalScriptName, + artifactVersion: attestation.artifactVersion, + specDigest: attestation.specDigest, + }; +} + +/** + * The retry delays this budget affords, computed up front rather than derived + * from the clock inside the loop. The clock bound below is the honest one, but + * it is injectable, and an injected clock that never advances would otherwise + * turn a permanent mismatch into an unbounded loop. Two independent bounds mean + * neither can be the only thing standing between a stuck route and a spin. + */ +function retryDelaySchedule( + budgetMs: number, + initialDelayMs: number, + maxDelayMs: number, +): readonly number[] { + const delays: number[] = []; + let delayMs = initialDelayMs; + let scheduledMs = 0; + while (scheduledMs + delayMs <= budgetMs) { + delays.push(delayMs); + scheduledMs += delayMs; + delayMs = Math.min(delayMs * 2, maxDelayMs); + } + return delays; +} + +/** + * Attest what is routed, waiting out the provider's own convergence, and fail + * closed when it does not converge. + * + * Every promote path needs this same wait for the same reason: a hostname-to- + * script mapping is eventually consistent, so the first read after a promote + * can legitimately still answer with the release that promote just replaced. + * Attesting once would make that ordinary lag look like drift. The wait lives + * here, once, rather than in each caller, so no promote path can be written + * without it. + * + * It is a wait, not a retry loop around a broken provider: an unconverged read + * — a stale routed release, a route mapping that has not landed yet, a routed + * artifact whose digest binding is not visible — is retried inside the budget, + * and anything else propagates immediately. When the budget runs out the last + * observation is raised as `ActiveRouteAttestationError`, never swallowed and + * never accepted as a match. + * + * A route that already matches costs exactly one attestation; the backoff only + * ever pays for a route that has not settled. + */ +export async function attestConvergedActiveRoute( + backend: ProvisioningBackend, + spec: DeploymentSpec, + expected: ActiveRouteExpectation, + options: AttestConvergedActiveRouteOptions = {}, +): Promise { + const clock = options.clock ?? Date.now; + const sleep = + options.sleep ?? + ((delayMs: number) => + new Promise((resolve) => { + setTimeout(resolve, delayMs); + })); + const budgetMs = positiveInteger( + options.convergenceBudgetMs ?? DEFAULT_CONVERGENCE_BUDGET_MS, + 'convergenceBudgetMs', + ); + const initialDelayMs = positiveInteger( + options.initialRetryDelayMs ?? DEFAULT_INITIAL_RETRY_DELAY_MS, + 'initialRetryDelayMs', + ); + const maxDelayMs = positiveInteger( + options.maxRetryDelayMs ?? DEFAULT_MAX_RETRY_DELAY_MS, + 'maxRetryDelayMs', + ); + if (maxDelayMs < initialDelayMs) { + throw new Error('maxRetryDelayMs must not be below initialRetryDelayMs'); + } + const delays = retryDelaySchedule(budgetMs, initialDelayMs, maxDelayMs); + const startedAt = clock(); + let lastAttestation: ActiveRouteAttestation | undefined; + let lastRefusal: ActiveRouteAttestationError | undefined; + for (let attempt = 0; ; attempt += 1) { + lastAttestation = undefined; + lastRefusal = undefined; + try { + const attestation = await backend.attestActiveRoute(spec); + if (attestationMatches(attestation, expected)) return attestation; + lastAttestation = attestation; + } catch (error) { + if (!(error instanceof ActiveRouteAttestationError)) throw error; + lastRefusal = error; + } + const delayMs = delays[attempt]; + if (delayMs === undefined) break; + if (clock() - startedAt + delayMs > budgetMs) break; + await sleep(delayMs); + } + const unconverged = `active route for '${spec.tenantTag}:${spec.environment}' did not converge to specification digest '${expected.specDigest}' within ${budgetMs}ms`; + if (lastAttestation) { + throw new ActiveRouteAttestationError( + `${unconverged}: '${spec.routeHostname}' still routes artifact '${lastAttestation.artifactVersion}' of '${lastAttestation.physicalScriptName}'`, + observedFrom(lastAttestation), + { attestation: lastAttestation }, + ); + } + // The ?? fallbacks are defensive: the loop always attempts once, so one of + // lastAttestation / lastRefusal is always set. + throw new ActiveRouteAttestationError( + `${unconverged}: ${lastRefusal?.message ?? 'no attestation was produced'}`, + lastRefusal?.observed ?? {}, + { cause: lastRefusal }, + ); +} + +/** + * Read what a persisted deployment is actually serving and say whether it is + * what the record desires. + * + * Single-shot on purpose: convergence backoff exists to wait out a promote this + * process just performed, and a host reading state performed none. Waiting here + * would only spend the provider budget re-asking a question whose answer has + * already settled. + * + * Reports rather than enforces. A disagreement comes back as + * `matchesDesired: false` with the attestation that proves it, because a host + * rendering fleet state wants to show drift, not fail on it. Only a provider + * that cannot be attested at all throws. + * + * NOT for per-request status rendering. Each single-shot call spends two + * provider reads on the plain-Worker backend or three on Workers for Platforms, + * against a default 1,100-request-per-five-minute account-wide budget shared by + * every provisioning operation. A status surface with any traffic at all must + * cache these results rather than attest per request. + */ +export async function attestFleetRecordActiveRoute(input: { + readonly record: FleetRecord; + readonly backend: ProvisioningBackend; + readonly spec: DeploymentSpec; +}): Promise< + Readonly<{ + attestation: ActiveRouteAttestation; + matchesDesired: boolean; + }> +> { + const attestation = await input.backend.attestActiveRoute(input.spec); + return { + attestation, + matchesDesired: attestationMatches(attestation, { + specDigest: input.record.desiredSpecDigest, + artifactVersion: input.record.artifactVersion, + }), + }; +} diff --git a/packages/fleet-control/src/cloudflare-client.ts b/packages/fleet-control/src/cloudflare-client.ts index dce7f3f..4434d4b 100644 --- a/packages/fleet-control/src/cloudflare-client.ts +++ b/packages/fleet-control/src/cloudflare-client.ts @@ -5,6 +5,7 @@ import { createHash } from 'node:crypto'; import Cloudflare from 'cloudflare'; import { toFile } from 'cloudflare/uploads'; import PQueue from 'p-queue'; +import { ActiveRouteAttestationError } from './active-route.js'; import { canonicalApplicationBindings } from './application-bindings.js'; import type { CloudflareApiRateCoordinator } from './cloudflare-rate-coordinator.js'; import { @@ -31,6 +32,7 @@ import type { DeploymentSpec, ExternalMutationFence, FleetResourceInventory, + PlainWorkerRouteApi, PromotionGuard, ProviderBindingIdentity, ScriptInventoryTarget, @@ -126,15 +128,17 @@ export interface OrdinaryWorkerFootprint { const SCRIPT_INVENTORY_PREFIX = '__anchorage_script__:'; const FLEET_SCRIPT_TAG = 'fleet:anchorage'; +type ProviderDeployment = + | Readonly<{ + versions?: readonly Readonly<{ + percentage?: unknown; + version_id?: unknown; + }>[]; + }> + | undefined; + function exactActiveVersionId( - deployment: - | Readonly<{ - versions?: readonly Readonly<{ - percentage?: unknown; - version_id?: unknown; - }>[]; - }> - | undefined, + deployment: ProviderDeployment, context: string, ): string { if (!deployment || !Array.isArray(deployment.versions)) { @@ -165,6 +169,52 @@ function exactActiveVersionId( return onlyVersion.version_id; } +/** + * The traffic split as the provider reported it, for a refusal to carry. Only + * well-formed entries survive: a malformed one is exactly what + * `exactActiveVersionId` already refused over, and inventing a shape for it + * would put a fabricated percentage into an operator-facing error. + */ +function observedTrafficSplit( + deployment: ProviderDeployment, +): readonly Readonly<{ artifactVersion: string; percentage: number }>[] { + return (deployment?.versions ?? []).flatMap((version) => + typeof version.version_id === 'string' && + typeof version.percentage === 'number' && + Number.isFinite(version.percentage) + ? [ + { + artifactVersion: version.version_id, + percentage: version.percentage, + }, + ] + : [], + ); +} + +/** + * `exactActiveVersionId` with its refusal restated as an attestation refusal + * carrying the split. The rule is unchanged and deliberately not relaxed: one + * version at 100% or nothing, never the version with the largest share. + */ +function attestedActiveVersionId( + deployment: ProviderDeployment, + scriptName: string, +): string { + try { + return exactActiveVersionId(deployment, `ordinary Worker '${scriptName}'`); + } catch (cause) { + throw new ActiveRouteAttestationError( + cause instanceof Error ? cause.message : String(cause), + { + routedScriptName: scriptName, + trafficSplit: observedTrafficSplit(deployment), + }, + { cause }, + ); + } +} + function tagValue(tags: readonly string[], prefix: string): string | undefined { return tags.find((tag) => tag.startsWith(prefix))?.slice(prefix.length); } @@ -381,7 +431,7 @@ async function hashExport( return { sha256: hash.digest('hex'), size }; } -export class CloudflareProvisioningClient { +export class CloudflareProvisioningClient implements PlainWorkerRouteApi { readonly #accountId: string; readonly #apiToken: string; readonly #dispatchNamespace: string; @@ -2473,6 +2523,47 @@ export class CloudflareProvisioningClient { ); } + async inspectActiveWorkerRoute(scriptName: string): Promise< + | Readonly<{ + artifactVersion: string; + specDigest: string | undefined; + }> + | undefined + > { + return this.#schedule(async () => { + let deploymentList: Awaited< + ReturnType + >; + try { + deploymentList = await this.#client.workers.scripts.deployments.list( + scriptName, + { account_id: this.#accountId }, + ); + } catch (error) { + if (isNotFound(error)) return undefined; + throw error; + } + const artifactVersion = attestedActiveVersionId( + deploymentList.deployments[0], + scriptName, + ); + const version = await this.#client.workers.scripts.versions.get( + artifactVersion, + { account_id: this.#accountId, script_name: scriptName }, + ); + const specDigest = (version.resources.bindings ?? []).flatMap( + (binding) => + binding.type === 'plain_text' && binding.name === 'FLEET_SPEC_DIGEST' + ? [binding.text] + : [], + )[0]; + return { + artifactVersion, + specDigest: typeof specDigest === 'string' ? specDigest : undefined, + }; + }); + } + async inspectOrdinaryWorkerFootprint( scriptName: string, ): Promise { diff --git a/packages/fleet-control/src/fleet.ts b/packages/fleet-control/src/fleet.ts index f65d077..b19b9e8 100644 --- a/packages/fleet-control/src/fleet.ts +++ b/packages/fleet-control/src/fleet.ts @@ -1,5 +1,6 @@ // SPDX-License-Identifier: Apache-2.0 +import type { AttestConvergedActiveRouteOptions } from './active-route.js'; import { applicationBindingTopology, DEPLOYMENT_PLATFORM_VARIABLE_NAMES, @@ -32,6 +33,7 @@ import { assertPlatformDurableObjectHistory, reconcilePersistedDatabase, } from './provision.js'; +import { settlePromotedRoute } from './settlement.js'; import { deploymentSpecDigest } from './spec-digest.js'; import type { DeploymentEgressPolicy, @@ -44,6 +46,7 @@ import type { FleetInventoryFinding, FleetRecord, FleetResourceInventory, + FleetSettlementHost, FleetStateLease, FleetStateStore, LiveDeployment, @@ -1357,6 +1360,30 @@ export async function migrateFleet(options: { readonly finalizedStateProviderFor?: ( record: FleetRecord, ) => FinalizedOrdinaryStateProvider | undefined; + /** + * The host to hand each settled promotion to, per deployment. + * + * Optional, and its absence changes nothing about correctness: every promote + * path attests what it published whether or not a host is settling, because + * checking its own work is the package's obligation rather than a service it + * performs for a caller. + * + * `provisionDeployment` deliberately consults nothing like this. A first + * deploy returns synchronously to the caller that asked for it, so the host + * already knows the moment it went live and can settle after the call using + * `attestFleetRecordActiveRoute`; an in-lease settlement point there would + * add a callback into the critical section to tell a caller something it is + * about to be told anyway. + */ + readonly settlementFor?: ( + record: FleetRecord, + ) => FleetSettlementHost | undefined; + /** + * Tuning for the convergence wait each post-promote attestation performs. + * The defaults suit every provider this package targets; a caller overrides + * them to bound the wait differently or to drive it from an injected clock. + */ + readonly routeAttestation?: AttestConvergedActiveRouteOptions; readonly clock?: () => number; }): Promise { const canaryOrder = new Map( @@ -1374,6 +1401,10 @@ export async function migrateFleet(options: { `${b.tenantTag}:${b.environment}`, ); }); + const attestationOptions: AttestConvergedActiveRouteOptions = { + clock: options.clock ?? Date.now, + ...options.routeAttestation, + }; const updated: FleetRecord[] = []; for (const record of ordered) { const next = await options.store.withDeploymentLease( @@ -1613,6 +1644,31 @@ export async function migrateFleet(options: { lease, activeArtifactVersion(stored), ); + // The steady-state path: an unchanged deployment reconciled again. + // It re-promotes because a crash could have left the route behind, + // so it must re-attest — but it must not re-settle, or a fleet on a + // reconcile schedule would settle forever. + const convergence = await settlePromotedRoute({ + backend, + spec, + record: stored, + entry: 'ready-convergence', + target: stored.activeRelease, + prior: stored.rollbackRelease, + expectedSpecDigest: targetDigest, + expectedArtifactVersion: activeArtifactVersion(stored), + settlementHost: options.settlementFor?.(stored), + attestation: attestationOptions, + skipWhenAlreadySettled: true, + }); + if (convergence.settled) { + stored = { + ...stored, + settledSettlementKey: convergence.settlementKey, + updatedAt: new Date((options.clock ?? Date.now)()).toISOString(), + }; + await lease.put(stored); + } return retireCommittedRelease( backend, spec, @@ -1863,6 +1919,18 @@ export async function migrateFleet(options: { platformMigrationRelease, 'platform-only settlement', ); + const platformSettlement = await settlePromotedRoute({ + backend, + spec, + record: migrationRecord, + entry: 'platform-only', + target: platformMigrationRelease, + prior: migrationRecord.rollbackRelease, + expectedSpecDigest: targetDigest, + expectedArtifactVersion: platformMigrationRelease.artifactVersion, + settlementHost: options.settlementFor?.(migrationRecord), + attestation: attestationOptions, + }); const settled = { ...migrationRecord }; delete settled.migrationIntent; const migrated: FleetRecord = { @@ -1870,13 +1938,36 @@ export async function migrateFleet(options: { phase: 'ready', platformTarget: platformMigrationTarget, outboundPolicy: platformMigrationTarget.outboundPolicy, + ...(platformSettlement.settled + ? { settledSettlementKey: platformSettlement.settlementKey } + : {}), updatedAt: new Date((options.clock ?? Date.now)()).toISOString(), }; await lease.put(migrated); return migrated; } await lease.assertOwned(); - await backend.seedDeploymentIdentity(database, stored.tenantTag, lease); + // Re-stamping a database this deployment already owns: the ownership + // sentinel short-circuits, and the only thing that can still happen is + // the fence row being CREATED where none exists. + // + // 'open' is hard-coded, and migrateFleet takes no fence option, for one + // reason: the deployment being migrated is `ready` or `migrating` — it + // is EXECUTING right now. A pre-0.20 database has no fence row and + // therefore reads as open; materializing that row must record what the + // deployment already IS, not impose something new. Seeding + // 'migration-locked' here would silently stop a live deployment in the + // middle of its own migration. Closing a fence is an operator action + // through POST /admin/execution-fence, never a side effect of a + // schema pass. + await backend.seedDeploymentIdentity( + database, + stored.tenantTag, + lease, + { + initialExecutionFenceState: 'open', + }, + ); const pendingMigrations = spec.migrations.filter( (candidate) => candidate.version > migrationRecord.schemaVersion, ); @@ -2192,6 +2283,18 @@ export async function migrateFleet(options: { 'promoted release has no exact persisted binding topology', ); } + const migrationSettlement = await settlePromotedRoute({ + backend, + spec, + record: migrationRecord, + entry: 'migration', + target: committedTargetRelease, + prior: rollbackRelease, + expectedSpecDigest: targetDigest, + expectedArtifactVersion: live.artifactVersion, + settlementHost: options.settlementFor?.(migrationRecord), + attestation: attestationOptions, + }); const settled = { ...migrationRecord }; delete settled.pendingRelease; delete settled.migrationPriorRelease; @@ -2240,6 +2343,9 @@ export async function migrateFleet(options: { spec, migrationRecord.applicationResources ?? [], ), + ...(migrationSettlement.settled + ? { settledSettlementKey: migrationSettlement.settlementKey } + : {}), updatedAt: new Date((options.clock ?? Date.now)()).toISOString(), }; await lease.put(migrated); @@ -2265,6 +2371,13 @@ export async function rollbackExternalRelease(options: { readonly rollbackSpec: DeploymentSpec; readonly secrets: DeploymentSecrets; readonly finalizedStateProvider?: FinalizedOrdinaryStateProvider; + /** + * The host to hand the reversal to. Singular because a rollback names one + * deployment, where `migrateFleet` sweeps many. + */ + readonly settlement?: FleetSettlementHost; + /** Tuning for the convergence wait the post-promote attestation performs. */ + readonly routeAttestation?: AttestConvergedActiveRouteOptions; readonly clock?: () => number; }): Promise { const { store, backend, currentSpec, rollbackSpec, secrets } = options; @@ -2534,6 +2647,24 @@ export async function rollbackExternalRelease(options: { ); assertExternalReleaseArtifactVersion(live, target, 'rollback settlement'); const nextRollback = stored.activeRelease; + // `prior` is the release being ABANDONED here, not the one replaced. + // A host reversing its own effects needs the snapshot traffic just left, + // and on this path that is the release that was active on entry. + const rollbackSettlement = await settlePromotedRoute({ + backend, + spec: rollbackSpec, + record: intent, + entry: 'rollback', + target, + prior: nextRollback, + expectedSpecDigest: target.specDigest, + expectedArtifactVersion: target.artifactVersion, + settlementHost: options.settlement, + attestation: { + clock: options.clock ?? Date.now, + ...options.routeAttestation, + }, + }); const settled = { ...intent }; delete settled.pendingRelease; const rolledBack: FleetRecord = { @@ -2551,6 +2682,9 @@ export async function rollbackExternalRelease(options: { rollbackSpec, stored.applicationResources ?? [], ), + ...(rollbackSettlement.settled + ? { settledSettlementKey: rollbackSettlement.settlementKey } + : {}), updatedAt: new Date((options.clock ?? Date.now)()).toISOString(), }; await lease.put(rolledBack); diff --git a/packages/fleet-control/src/index.ts b/packages/fleet-control/src/index.ts index 59bfe08..5c9c268 100644 --- a/packages/fleet-control/src/index.ts +++ b/packages/fleet-control/src/index.ts @@ -1,5 +1,12 @@ // SPDX-License-Identifier: Apache-2.0 +export { + ActiveRouteAttestationError, + type ActiveRouteExpectation, + type AttestConvergedActiveRouteOptions, + attestConvergedActiveRoute, + attestFleetRecordActiveRoute, +} from './active-route.js'; export { applicationBindingTopology, applicationR2Bindings, @@ -82,6 +89,7 @@ export { provisionDeployment, } from './provision.js'; export { generateDeploymentSecrets } from './secrets.js'; +export { fleetSettlementKey } from './settlement.js'; export { deploymentSpecDigest } from './spec-digest.js'; export { D1FleetStateStore, @@ -89,6 +97,7 @@ export { type FleetStateDatabase, } from './state-store.js'; export { + type ActiveRouteAttestation, type ApplicationBindingTopology, type ApplicationR2Binding, type ApplicationR2BucketSnapshot, @@ -117,11 +126,18 @@ export { type FleetInventoryFinding, type FleetRecord, type FleetResourceInventory, + type FleetSettlementContext, + type FleetSettlementEntry, + type FleetSettlementHost, type FleetStateLease, type FleetStateStore, type ForceDecommissionStep, + type InitialExecutionFenceState, type LiveDeployment, type MaintenanceHealth, + type ObservedActiveRoute, + type PlainWorkerCustomDomain, + type PlainWorkerRouteApi, type PlatformPlaneLease, type PlatformPlaneResourceSet, type PlatformPlaneStateStore, @@ -136,6 +152,7 @@ export { type ProvisioningResult, type R2Jurisdiction, type ScriptInventoryTarget, + type SeedDeploymentIdentityOptions, type TrustedWorkerArtifact, type WorkerModule, type WorkerZoneRoute, @@ -159,11 +176,7 @@ export { WorkersForPlatformsBackendSwitchProvider, type WorkersForPlatformsBackendSwitchProviderOptions, } from './workers-for-platforms-backend-switch-provider.js'; -export { - type PlainWorkerCustomDomain, - type PlainWorkerRouteApi, - WranglerLoopBackend, -} from './wrangler-loop-backend.js'; +export { WranglerLoopBackend } from './wrangler-loop-backend.js'; export { type CommandResult, type CommandRunner, diff --git a/packages/fleet-control/src/provision.ts b/packages/fleet-control/src/provision.ts index 6ee7146..5b7a58a 100644 --- a/packages/fleet-control/src/provision.ts +++ b/packages/fleet-control/src/provision.ts @@ -1,5 +1,12 @@ // SPDX-License-Identifier: Apache-2.0 +import { assertInitialExecutionFenceState } from '@proofoftech/flowsafe/deployment-identity-protocol'; + +import { + type AttestConvergedActiveRouteOptions, + attestConvergedActiveRoute, + PENDING_ARTIFACT_VERSION, +} from './active-route.js'; import { applicationBindingTopology, applicationR2Bindings, @@ -44,6 +51,7 @@ import type { FleetRecord, FleetStateLease, FleetStateStore, + InitialExecutionFenceState, MaintenanceHealth, ProvisioningBackend, ProvisioningPhase, @@ -78,7 +86,8 @@ function activeExternalRelease( ): import('./types.js').ExternalReleaseSnapshot | undefined { return ( record.activeRelease ?? - (record.backend === 'plain-worker' && record.artifactVersion !== 'pending' + (record.backend === 'plain-worker' && + record.artifactVersion !== PENDING_ARTIFACT_VERSION ? { physicalScriptName: record.scriptName, specDigest: record.desiredSpecDigest, @@ -161,7 +170,7 @@ function recordAt( databaseId: database.id, databaseName: database.name, schemaVersion: options.schemaVersion ?? 0, - artifactVersion: options.artifactVersion ?? 'pending', + artifactVersion: options.artifactVersion ?? PENDING_ARTIFACT_VERSION, desiredSpecDigest: deploymentSpecDigest(spec), durableObjectBindings: options.durableObjectBindings ?? [], applicationResources, @@ -301,7 +310,7 @@ export function assertExternalReleaseArtifactVersion( release: import('./types.js').ExternalReleaseSnapshot, context: string, ): void { - if (release.artifactVersion === 'pending') return; + if (release.artifactVersion === PENDING_ARTIFACT_VERSION) return; if ( !live || live.scriptName !== release.physicalScriptName || @@ -573,13 +582,50 @@ export interface ProvisionDeploymentOptions { readonly store: FleetStateStore; readonly spec: DeploymentSpec; readonly secrets: DeploymentSecrets; + /** + * The execution-fence state this deployment is born in. REQUIRED; no default. + * + * It lives on the OPTIONS and deliberately not on `DeploymentSpec`. The spec + * is the digested canonical description of a deployment + * (`deploymentSpecDigest`), and every stored record, drift audit, and + * migration decision compares against that digest — adding a field to it + * would change the digest of every deployment already in the fleet and + * present as fleet-wide drift, scheduling migrations for artifacts nobody + * touched. + * + * It is also provisioning-time-only by nature: after the first pass the fence + * is live operational state an operator moves through + * `POST /admin/execution-fence`, so `migrateFleet`, `rollbackExternalRelease` + * and `decommissionDeployment` neither take it nor need it — re-seeding is + * INSERT-if-absent and can only ever repair a missing row. + */ + readonly initialExecutionFenceState: InitialExecutionFenceState; readonly finalizedStateProvider?: FinalizedOrdinaryStateProvider; + /** + * Tuning for the convergence wait the ready-commit attestation performs. + * The defaults suit every provider this package targets. + */ + readonly routeAttestation?: AttestConvergedActiveRouteOptions; readonly clock?: () => number; } -export function provisionDeployment( +// `async` so the entry validation below REJECTS rather than throwing +// synchronously: every caller and every test treats this as a promise-returning +// function, and a synchronous throw would escape an unguarded `.catch()`. +export async function provisionDeployment( options: ProvisionDeploymentOptions, ): Promise { + // Validated HERE, before the lease and before a single provider call. The + // protocol validates it too, but only when the seeding statements are built — + // which is after `database-created`, so a garbage value would have created a + // Worker and a D1 database first and then failed, leaving a half-provisioned + // deployment for an operator to reconcile over a typo. Nothing about this + // value depends on anything the lease reads, so the cheapest place to refuse + // it is the entry. + assertInitialExecutionFenceState( + options.initialExecutionFenceState, + 'provisionDeployment', + ); return options.store.withDeploymentLease( options.spec.tenantTag, options.spec.environment, @@ -841,7 +887,9 @@ async function provisionDeploymentUnderLease( if (record.phase === 'database-created') { await lease.assertOwned(); - await backend.seedDeploymentIdentity(database, spec.tenantTag, lease); + await backend.seedDeploymentIdentity(database, spec.tenantTag, lease, { + initialExecutionFenceState: options.initialExecutionFenceState, + }); databaseOwnershipProven = true; record = { ...record, @@ -1007,7 +1055,7 @@ async function provisionDeploymentUnderLease( secrets, record.platformResources, lease, - immutableExternal ? 'pending' : undefined, + immutableExternal ? PENDING_ARTIFACT_VERSION : undefined, record.applicationBindings, ); workerCreated = deployed.created; @@ -1086,7 +1134,8 @@ async function provisionDeploymentUnderLease( record.platformResources, lease, immutableExternal - ? (record.pendingRelease?.artifactVersion ?? 'pending') + ? (record.pendingRelease?.artifactVersion ?? + PENDING_ARTIFACT_VERSION) : undefined, record.applicationBindings, ); @@ -1255,6 +1304,20 @@ async function provisionDeploymentUnderLease( if (!maintenance.armed) { throw new Error('maintenance is unarmed before ready commit'); } + // The committed artifact version is the ROUTED one, not the one an + // inspection reported. Inspection deliberately pins the candidate it was + // asked about, so committing its answer would record a version this + // deployment merely uploaded as the version it serves. + const attestation = await attestConvergedActiveRoute( + backend, + spec, + { + specDigest: record.desiredSpecDigest, + artifactVersion: + record.pendingRelease?.artifactVersion ?? record.artifactVersion, + }, + { clock, ...options.routeAttestation }, + ); const readyRecord = { ...record }; if (readyRecord.pendingRelease) { readyRecord.activeRelease = readyRecord.pendingRelease; @@ -1263,7 +1326,7 @@ async function provisionDeploymentUnderLease( record = { ...readyRecord, phase: 'ready', - artifactVersion: live.artifactVersion, + artifactVersion: attestation.artifactVersion, durableObjectTag: targetDurableObjectTag(spec), durableObjectBindings: live.durableObjectBindings, updatedAt: nowIso(clock), @@ -1424,10 +1487,17 @@ async function cleanupDeploymentArtifactsUnderLease( ); } await lease.assertOwned(); + // A freshness PROOF, not a provisioning: this database is stamped only so + // the read-back below can show it was empty, and it is deleted three lines + // later. The fence state is therefore hard-coded rather than taken from the + // caller — cleanup has no provisioning options to take it from — and it is + // 'migration-locked' because a database that survives a failed delete must + // never come back as one that executes. await backend.seedDeploymentIdentity( reservedDatabase, record.tenantTag, lease, + { initialExecutionFenceState: 'migration-locked' }, ); const seededOwner = await backend.readDeploymentIdentity( reservedDatabase, diff --git a/packages/fleet-control/src/settlement.ts b/packages/fleet-control/src/settlement.ts new file mode 100644 index 0000000..16b5ea0 --- /dev/null +++ b/packages/fleet-control/src/settlement.ts @@ -0,0 +1,170 @@ +// SPDX-License-Identifier: Apache-2.0 + +import { createHash } from 'node:crypto'; +import { + type AttestConvergedActiveRouteOptions, + attestConvergedActiveRoute, + PENDING_ARTIFACT_VERSION, +} from './active-route.js'; +import type { + ActiveRouteAttestation, + DeploymentSpec, + ExternalReleaseSnapshot, + FleetRecord, + FleetSettlementEntry, + FleetSettlementHost, + ProvisioningBackend, +} from './types.js'; + +/** + * Identifies a settlement by the deployment and the release that settled. + * + * Deliberately excludes the entry, the attempt, and the clock. A migration that + * crashes after `settle()` and re-enters must produce the same key or the + * at-least-once contract is worthless, and a rollback that returns a + * deployment to a release it settled before is settling that same release + * again — the same key is the honest answer, not a collision. + */ +export function fleetSettlementKey(input: { + readonly tenantTag: string; + readonly environment: string; + readonly specDigest: string; + readonly artifactVersion: string; +}): string { + return createHash('sha256') + .update( + JSON.stringify({ + tenantTag: input.tenantTag, + environment: input.environment, + specDigest: input.specDigest, + artifactVersion: input.artifactVersion, + }), + ) + .digest('hex'); +} + +/** + * Resolves the snapshot a settlement reports as its target. + * + * Two adjustments, both narrow. A planned release still carrying the `pending` + * sentinel takes the attested version. A key built from the constant `pending` + * would repeat across retries of that pass, but it would not match the key the + * same release produces once its provider version is known; a later converge + * would therefore settle that already-settled release again. + * + * The settlement key has four inputs. `tenantTag` and `environment` are + * control-plane identity from the record. With a persisted target snapshot, + * `specDigest` comes from that snapshot and `artifactVersion` does too, unless + * its `pending` sentinel is replaced by the attestation. Without a snapshot, + * both release fields come from the attestation. + * + * A deployment with no release snapshots at all — the plain backend keeps + * none — gets one synthesized, and it is a MIXED record rather than a provider + * reading. `physicalScriptName`, `specDigest`, and `artifactVersion` are + * attested. `releaseSchemaVersion` and `application` are copied from the + * control-plane record because no provider read reports them; they are what + * this deployment believes it deployed, and a host must not treat them as + * attested. + */ +function settledTargetRelease( + target: ExternalReleaseSnapshot | undefined, + attestation: ActiveRouteAttestation, + record: FleetRecord, +): ExternalReleaseSnapshot { + if (!target) { + return { + physicalScriptName: attestation.physicalScriptName, + specDigest: attestation.specDigest, + artifactVersion: attestation.artifactVersion, + releaseSchemaVersion: record.schemaVersion, + ...(record.applicationBindings + ? { application: record.applicationBindings } + : {}), + }; + } + return target.artifactVersion === PENDING_ARTIFACT_VERSION + ? { ...target, artifactVersion: attestation.artifactVersion } + : target; +} + +export interface SettlePromotedRouteInput { + readonly backend: ProvisioningBackend; + readonly spec: DeploymentSpec; + /** The record as it stands after the promotion, before the settling write. */ + readonly record: FleetRecord; + readonly entry: FleetSettlementEntry; + /** The release expected to be serving; absent where none is persisted. */ + readonly target: ExternalReleaseSnapshot | undefined; + readonly prior: ExternalReleaseSnapshot | undefined; + /** The digest the route must be serving. Never taken from the provider. */ + readonly expectedSpecDigest: string; + /** The artifact expected, or `pending` when none is known yet. */ + readonly expectedArtifactVersion: string; + readonly settlementHost: FleetSettlementHost | undefined; + readonly attestation: AttestConvergedActiveRouteOptions | undefined; + /** + * Set on the convergence entry only. That entry is re-entered on every + * routine reconcile of an unchanged deployment, so re-firing settlement there + * would bill a fleet for standing still. Every other entry follows a change + * and settles unconditionally, which is what keeps at-least-once honest + * across a crash between `settle()` and the write that records it. + */ + readonly skipWhenAlreadySettled?: true; +} + +export interface SettledPromotedRoute { + readonly attestation: ActiveRouteAttestation; + readonly settlementKey: string; + /** Whether `settle()` was actually called on this pass. */ + readonly settled: boolean; +} + +/** + * Attest what the promotion published, then settle it — in that order, and only + * in that order. + * + * Attestation is unconditional and runs whether or not a host settles: proving + * the promotion took effect is the package's own obligation, and gating it on + * an optional callback would mean a fleet without one never checks its own + * work. Settlement is what is optional, and it runs only once attestation has + * both succeeded and matched, so a host is never told a release is live on the + * strength of a promotion this process merely issued. + */ +export async function settlePromotedRoute( + input: SettlePromotedRouteInput, +): Promise { + const attestation = await attestConvergedActiveRoute( + input.backend, + input.spec, + { + specDigest: input.expectedSpecDigest, + artifactVersion: input.expectedArtifactVersion, + }, + input.attestation ?? {}, + ); + const target = settledTargetRelease(input.target, attestation, input.record); + const settlementKey = fleetSettlementKey({ + tenantTag: input.record.tenantTag, + environment: input.record.environment, + specDigest: target.specDigest, + artifactVersion: target.artifactVersion, + }); + const alreadySettled = input.record.settledSettlementKey === settlementKey; + if ( + !input.settlementHost || + (input.skipWhenAlreadySettled && alreadySettled) + ) { + return { attestation, settlementKey, settled: false }; + } + await input.settlementHost.settle({ + tenantTag: input.record.tenantTag, + environment: input.record.environment, + attestation, + target, + ...(input.prior ? { prior: input.prior } : {}), + entry: input.entry, + settlementKey, + alreadySettled, + }); + return { attestation, settlementKey, settled: true }; +} diff --git a/packages/fleet-control/src/state-store.ts b/packages/fleet-control/src/state-store.ts index 1d5fe4b..36f0a1f 100644 --- a/packages/fleet-control/src/state-store.ts +++ b/packages/fleet-control/src/state-store.ts @@ -144,39 +144,49 @@ const FLEET_ROW_COLUMNS = [ 'database_export_location', 'database_export_sha256', 'database_export_size', + 'settled_settlement_key', 'updated_at', ] as const; -function isDuplicateBackendSwitchColumn(error: unknown): boolean { +/** + * Nullable TEXT columns added to a table that already shipped, in the order + * they were added. Each is created by ALTER on an existing database and by the + * CREATE above on a new one, and each is asserted present afterwards: a column + * that silently failed to appear would not fail a write, it would drop the + * value on every write. + */ +export const ADDED_NULLABLE_TEXT_COLUMNS = [ + 'backend_switch_intent', + 'settled_settlement_key', +] as const; + +function isDuplicateColumnError( + error: unknown, + column: (typeof ADDED_NULLABLE_TEXT_COLUMNS)[number], +): boolean { + const duplicate = new RegExp(`duplicate column name:\\s*${column}\\b`, 'iu'); let current = error; const seen = new Set(); while (current instanceof Error && !seen.has(current)) { seen.add(current); - if ( - /duplicate column name:\s*backend_switch_intent\b/iu.test(current.message) - ) { - return true; - } + if (duplicate.test(current.message)) return true; current = current.cause; } return false; } -function assertBackendSwitchColumn( +function assertNullableTextColumn( columns: readonly Readonly>[], + name: string, ): void { - const column = columns.find( - (candidate) => candidate.name === 'backend_switch_intent', - ); + const column = columns.find((candidate) => candidate.name === name); if ( !column || String(column.type).toUpperCase() !== 'TEXT' || Number(column.notnull) !== 0 || Number(column.pk) !== 0 ) { - throw new Error( - 'fleet state backend_switch_intent column is absent or incompatible', - ); + throw new Error(`fleet state ${name} column is absent or incompatible`); } } @@ -887,6 +897,15 @@ function toRecord(row: Readonly>): FleetRecord { ) { throw new Error('fleet state row has invalid database_export_size'); } + const settledSettlementKey = row.settled_settlement_key; + if ( + settledSettlementKey !== null && + settledSettlementKey !== undefined && + (typeof settledSettlementKey !== 'string' || + !isSha256(settledSettlementKey)) + ) { + throw new Error('fleet state row has invalid settled_settlement_key'); + } const durableObjectBindings = JSON.parse( rowString(row, 'durable_object_bindings'), ) as unknown; @@ -1117,6 +1136,9 @@ function toRecord(row: Readonly>): FleetRecord { ...(databaseExportSize !== null && databaseExportSize !== undefined ? { databaseExportSize: Number(databaseExportSize) } : {}), + ...(typeof settledSettlementKey === 'string' && settledSettlementKey + ? { settledSettlementKey } + : {}), updatedAt: rowString(row, 'updated_at'), }; } @@ -1196,6 +1218,7 @@ export class D1FleetStateStore database_export_location TEXT, database_export_sha256 TEXT, database_export_size INTEGER, + settled_settlement_key TEXT, updated_at TEXT NOT NULL, PRIMARY KEY (tenant_tag, environment), UNIQUE (backend, script_name), @@ -1204,19 +1227,19 @@ export class D1FleetStateStore UNIQUE (route_hostname) )`); let fleetColumns = await this.#db.query(`PRAGMA table_info(${TABLE})`); - if ( - !fleetColumns.some((column) => column.name === 'backend_switch_intent') - ) { + for (const name of ADDED_NULLABLE_TEXT_COLUMNS) { + if (fleetColumns.some((column) => column.name === name)) continue; try { - await this.#db.execute( - `ALTER TABLE ${TABLE} ADD COLUMN backend_switch_intent TEXT`, - ); + await this.#db.execute(`ALTER TABLE ${TABLE} ADD COLUMN ${name} TEXT`); } catch (error) { - if (!isDuplicateBackendSwitchColumn(error)) throw error; + // A replica that added the same column between the read and the write. + if (!isDuplicateColumnError(error, name)) throw error; } fleetColumns = await this.#db.query(`PRAGMA table_info(${TABLE})`); } - assertBackendSwitchColumn(fleetColumns); + for (const name of ADDED_NULLABLE_TEXT_COLUMNS) { + assertNullableTextColumn(fleetColumns, name); + } await this.#db.execute(`UPDATE ${TABLE} SET backend_switch_intent = migration_intent, migration_intent = NULL @@ -1777,6 +1800,7 @@ export class D1FleetStateStore record.databaseExportLocation ?? null, record.databaseExportSha256 ?? null, record.databaseExportSize ?? null, + record.settledSettlementKey ?? null, record.updatedAt, ] as const; const upsertSql = `INSERT INTO ${TABLE} ( @@ -1789,13 +1813,13 @@ export class D1FleetStateStore durable_object_bindings, application_resources, application_bindings, route_hostname, phase, database_export_location, database_export_sha256, - database_export_size, updated_at + database_export_size, settled_settlement_key, updated_at ) SELECT CASE WHEN EXISTS ( SELECT 1 FROM ${LEASE_TABLE} WHERE tenant_tag = ? AND environment = ? AND owner_token = ? AND expires_at > ${DB_NOW_MS} ) THEN ? ELSE NULL END, - ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ? + ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ? WHERE true ON CONFLICT (tenant_tag, environment) DO UPDATE SET backend = excluded.backend, @@ -1828,6 +1852,7 @@ export class D1FleetStateStore database_export_location = excluded.database_export_location, database_export_sha256 = excluded.database_export_sha256, database_export_size = excluded.database_export_size, + settled_settlement_key = excluded.settled_settlement_key, updated_at = excluded.updated_at RETURNING tenant_tag, environment`; let results: readonly (readonly Readonly>[])[]; diff --git a/packages/fleet-control/src/types.ts b/packages/fleet-control/src/types.ts index 94b4294..5baaf85 100644 --- a/packages/fleet-control/src/types.ts +++ b/packages/fleet-control/src/types.ts @@ -1,5 +1,17 @@ // SPDX-License-Identifier: Apache-2.0 +import type { InitialExecutionFenceState } from '@proofoftech/flowsafe/deployment-identity-protocol'; + +/** + * The execution-fence state a freshly provisioned deployment is born in — + * 'open' or 'migration-locked'. + * + * Re-exported so a control plane names the choice in its own types without + * reaching into flowsafe's protocol subpath. Deliberately NOT part of + * `DeploymentSpec`: see `provisionDeployment`'s option for why. + */ +export type { InitialExecutionFenceState }; + export type ProvisioningBackendKind = 'plain-worker' | 'workers-for-platforms'; export interface WorkerModule { @@ -331,6 +343,17 @@ export interface FleetRecord { readonly databaseExportLocation?: string; readonly databaseExportSha256?: string; readonly databaseExportSize?: number; + /** + * The key of the last settlement this deployment completed. + * + * Persisted for one reason: the convergence entry is the steady-state path, + * so a fleet that reconciles on a schedule re-enters it forever. Without a + * durable marker every one of those converges would re-fire settlement on an + * unchanged deployment, and a host charging per settlement would charge per + * reconcile. Absent on any deployment that has not settled since this field + * existed, which reads as "settle once more", never as "already settled". + */ + readonly settledSettlementKey?: string; readonly updatedAt: string; } @@ -489,6 +512,111 @@ export interface ExternalMutationFence { assertOwned(): Promise; } +export interface PlainWorkerCustomDomain { + readonly id: string; + readonly hostname: string; + readonly service: string; +} + +export interface PlainWorkerRouteApi { + withMutationFence( + fence: ExternalMutationFence, + operation: () => Promise, + ): Promise; + queryDatabase( + databaseId: string, + sql: string, + bindings?: readonly string[], + ): Promise>[]>; + batchDatabase( + databaseId: string, + statements: readonly { + readonly sql: string; + readonly bindings?: readonly string[]; + }[], + ): Promise; + getDatabase?(databaseId: string): Promise; + deleteDatabase?(databaseId: string): Promise; + listWorkerDatabaseAttachments(databaseId: string): Promise< + readonly Readonly<{ + scriptName: string; + plane: 'ordinary' | 'dispatch'; + dispatchNamespace?: string; + }>[] + >; + listWorkerR2Attachments?(bucketName: string): Promise< + readonly Readonly<{ + scriptName: string; + plane: 'ordinary' | 'dispatch'; + dispatchNamespace?: string; + }>[] + >; + getR2Bucket?( + bucketName: string, + jurisdiction: R2Jurisdiction, + ): Promise; + createR2Bucket?( + resource: ApplicationR2Binding, + fence: ExternalMutationFence, + ): Promise; + assertR2BucketEmpty?(resource: ApplicationR2Binding): Promise; + deleteR2Bucket?( + resource: ApplicationR2Binding, + fence: ExternalMutationFence, + ): Promise; + /** + * The version currently taking all of an ordinary Worker's traffic, plus the + * fleet specification digest that version was built from. + * + * Narrower than the version read `inspect()` performs, deliberately: an + * attestation needs the routed version and one binding, and every provider + * call it makes is charged against the account-wide request window the rate + * coordinator fences. It also goes through the provider API rather than the + * wrangler CLI, which runs outside that coordinator entirely. + * + * Returns undefined when the script does not exist. Throws + * `ActiveRouteAttestationError` when it exists but no single version holds + * 100% of the traffic — that ambiguity is the refusal, not a tie to break. + */ + inspectActiveWorkerRoute(scriptName: string): Promise< + | Readonly<{ + artifactVersion: string; + specDigest: string | undefined; + }> + | undefined + >; + listCustomDomains(): Promise; + inspectOrdinaryWorkerFootprint(scriptName: string): Promise<{ + readonly scriptPresent: boolean; + readonly workersDevEnabled?: boolean; + readonly previewUrlsEnabled?: boolean; + readonly customDomains: readonly PlainWorkerCustomDomain[]; + readonly zoneRoutes: readonly WorkerZoneRoute[]; + }>; + listDurableObjectNamespaces(scriptName: string): Promise; + listOrdinaryWorkerSecretNames(scriptName: string): Promise; + deleteControlSecrets( + scriptName: string, + secretNames: readonly string[], + fence: ExternalMutationFence, + ): Promise; + attachCustomDomain( + target: { + readonly hostname: string; + readonly service: string; + }, + fence: ExternalMutationFence, + ): Promise; + detachCustomDomain( + domainId: string, + fence: ExternalMutationFence, + ): Promise; + disableOrdinaryWorkerPublicAccess( + scriptName: string, + fence: ExternalMutationFence, + ): Promise; +} + export interface FleetStateLease extends ExternalMutationFence { readonly tenantTag: string; readonly environment: string; @@ -557,6 +685,137 @@ export interface PlatformPlaneStateStore { ): Promise; } +/** The provisioning context `seedDeploymentIdentity` carries past the fence. */ +export interface SeedDeploymentIdentityOptions { + /** The execution-fence state this deployment is born in. Required. */ + readonly initialExecutionFenceState: InitialExecutionFenceState; +} + +/** + * What the provider actually reported when a route could not be attested. + * + * Every field is optional because an attestation failure is defined by what is + * MISSING. A Worker splitting traffic across two versions has percentages but + * no single routed version; an unrouted hostname has neither; a routed script + * built without the fleet digest binding has everything except `specDigest`. + * The refusal carries whichever of those the provider did answer, so an + * operator reading the failure sees the drift rather than a bare "mismatch". + */ +export interface ObservedActiveRoute { + readonly trafficSplit?: readonly Readonly<{ + artifactVersion: string; + percentage: number; + }>[]; + readonly routedScriptName?: string; + readonly artifactVersion?: string; + readonly specDigest?: string; +} + +/** + * Provider truth about which artifact is serving a deployment's hostname right + * now, and which fleet specification that artifact was built from. + * + * `physicalScriptName` is the script traffic ACTUALLY reaches, which is not + * necessarily the release the control plane expected — reporting the difference + * is the whole point. `observedAt` comes from the backend's injected clock, so + * a caller comparing two attestations is comparing one clock, not two. + */ +export interface ActiveRouteAttestation { + readonly specDigest: string; + readonly artifactVersion: string; + readonly physicalScriptName: string; + readonly source: 'workers-deployments' | 'dispatch-route'; + readonly observedAt: string; +} + +/** Which promote path settled, so a host can tell four arrivals apart. */ +export type FleetSettlementEntry = + | 'migration' + | 'platform-only' + | 'ready-convergence' + | 'rollback'; + +export interface FleetSettlementContext { + readonly tenantTag: string; + readonly environment: string; + /** Proof of what is routed, read after the promotion converged. */ + readonly attestation: ActiveRouteAttestation; + /** + * The release now serving traffic, and the only deployment identity + * `settle()` may depend on. + * + * That limit is forced, not stylistic: on the convergence entry the prior + * release has already been retired by the time any settlement point is + * reached, and the plain backend retains no prior release at all. A host that + * needed the outgoing release to compute what it settles would work on some + * entries and silently misbehave on others. + * + * On a backend that retains no release snapshots this is synthesized: its + * script name, specification digest, and artifact version are the + * attestation's, but `releaseSchemaVersion` and `application` are copied from + * the control-plane record — this deployment's belief about what it deployed, + * not something the provider confirmed. + */ + readonly target: ExternalReleaseSnapshot; + /** + * Optional-normal, and its meaning is defined per entry: on 'migration', + * 'platform-only', and 'ready-convergence' it is the release this one + * replaced, where the deployment still retains it; on 'rollback' it is the + * release being ABANDONED, so a host reversing its own effects reverses the + * right one. Absent whenever no prior release is retained. + */ + readonly prior?: ExternalReleaseSnapshot; + readonly entry: FleetSettlementEntry; + /** + * Identifies this settlement by what was settled — the deployment and the + * target release — not by when it happened. Every retry of the same + * settlement carries the same key, which is what makes at-least-once + * delivery safe to deduplicate on. + */ + readonly settlementKey: string; + /** + * True only when an earlier successful settling write durably recorded this + * exact key on the fleet record. False includes the re-fire window where + * `settle()` succeeded but that write was lost, so false is never proof of a + * first delivery. Hosts must deduplicate on `settlementKey` and may use this + * field only for logging or alerting. + */ + readonly alreadySettled: boolean; +} + +export interface FleetSettlementHost { + /** + * Called while the deployment lease is held, after the route attested and + * matched, and before the state write that records the settlement. + * + * AT-LEAST-ONCE and KEYED. A crash between this returning and that write + * replays it with the same `settlementKey`, so anything with an external + * effect — a charge, an entitlement, a notification — must be idempotent on + * that key. This package cannot make the callback and its own durable write + * atomic, so it guarantees the direction that fails safe: never settled + * without being attempted. + * + * NO CALLBACK TIMEOUT. By default, the lease renews on a five-minute + * heartbeat against a fifteen-minute TTL; both figures are configurable on + * the state store. This package imposes no timeout on `settle()`: it renews + * the lease for as long as the callback runs, so a hung callback that keeps + * renewing holds the lease indefinitely and blocks every other operation on + * the deployment, including decommission. Keep `settle()` well inside the + * default renewal interval and enqueue slow work. Renewal errors are + * inspected only after the callback returns; the heartbeat cannot interrupt + * or time it out. + * + * If the process dies mid-callback, the lease expires one TTL after the last + * successful renewal — fifteen minutes by default — and re-entry re-fires + * `settle()` under the same `settlementKey`. + * + * A throw propagates. The branch's durable state is left where a re-entry + * resumes it, and that re-entry re-attests and settles again under the same + * key. + */ + settle(context: FleetSettlementContext): Promise; +} + export interface ProvisioningBackend { readonly kind: ProvisioningBackendKind; readonly immutableExternalArtifacts?: true; @@ -567,10 +826,28 @@ export interface ProvisioningBackend { spec: DeploymentSpec, fence: ExternalMutationFence, ): Promise; + /** + * Stamp the database's ownership sentinel and seed its initial execution + * fence row. + * + * `initialExecutionFenceState` is a REQUIRED option rather than backend + * configuration because it is a per-provisioning decision: the same control + * plane brings ordinary deployments up open and migration targets up locked. + * It carries no default anywhere on this path — a migration target that came + * up open would be executing exactly when it must not. + * + * It rides an OPTIONS object rather than a fourth positional because this + * method is implemented by every backend and faked by every test that builds + * one: the next piece of provisioning context to reach the seeding protocol + * would otherwise mean a fifth positional and the same fan-out again, and + * positional four and five of a five-argument call are exactly where a + * transposed argument compiles and provisions the wrong thing. + */ seedDeploymentIdentity( database: DatabaseReference, tenantTag: string, fence: ExternalMutationFence, + options: SeedDeploymentIdentityOptions, ): Promise; readDeploymentIdentity( database: DatabaseReference, @@ -658,6 +935,28 @@ export interface ProvisioningBackend { maintenanceAdminSecret: string, expectedArtifactVersion: string | undefined, ): Promise; + /** + * Provider truth about what is ROUTED, never desired state. + * + * `inspect()` answers a different question — "does the deployment the control + * plane WANTS exist and match?" — and deliberately pins a staged candidate + * that is receiving none of the traffic, so a converge can compare against it + * before promoting it. That makes it the wrong instrument for asking what is + * serving requests: a candidate uploaded and never promoted answers as though + * it were live. This method answers only the second question and never pins a + * candidate; the two views stay separate on purpose. + * + * Read-only and lease-free. Provider GET/HEAD requests bypass the external + * mutation fence, so a caller holding no deployment lease may attest, and a + * caller already inside one does not nest a second. + * + * Refuses rather than guessing: a Worker splitting traffic across versions, + * a hostname routed nowhere, or a routed artifact carrying no fleet + * specification digest all throw `ActiveRouteAttestationError` carrying what + * the provider did report. There is no highest-percentage fallback — an + * attestation is either unambiguous or it is a refusal. + */ + attestActiveRoute(spec: DeploymentSpec): Promise; removeTraffic( spec: DeploymentSpec, retainedReleases: readonly ExternalReleaseSnapshot[] | undefined, diff --git a/packages/fleet-control/src/workers-for-platforms-backend.ts b/packages/fleet-control/src/workers-for-platforms-backend.ts index 71ece8a..9aa629b 100644 --- a/packages/fleet-control/src/workers-for-platforms-backend.ts +++ b/packages/fleet-control/src/workers-for-platforms-backend.ts @@ -11,11 +11,14 @@ import { mintAsymmetricMaintenanceCapability, verifyMaintenanceReceipt, } from '@proofoftech/flowsafe/host-kit'; +import { ActiveRouteAttestationError } from './active-route.js'; import { applicationSecretNames, applicationSecretValues, } from './application-bindings.js'; +import { isSha256 } from './deployment-context.js'; import { WorkerDeploymentError } from './deployment-error.js'; +import { parseHostRoutingTarget } from './host-routing.js'; import { readMaintenanceHealth } from './maintenance-health.js'; import { applyMigrationsWithLedger, @@ -39,6 +42,7 @@ import { import { assertProviderBindingIdentitiesMatchInspection } from './provider-binding-inventory.js'; import { deploymentSpecDigest } from './spec-digest.js'; import type { + ActiveRouteAttestation, D1Migration, DatabaseExport, DatabaseReference, @@ -58,6 +62,7 @@ import type { ProviderBindingIdentity, ProvisioningBackend, ScriptInventoryTarget, + SeedDeploymentIdentityOptions, } from './types.js'; const RELEASE_DIGEST_LENGTH = 48; @@ -343,6 +348,7 @@ export class WorkersForPlatformsBackend implements ProvisioningBackend { readonly #hostRoutingKvId: string; readonly #auditQueueName?: string; readonly #maintenanceRequestTimeoutMs: number; + readonly #clock: () => number; readonly #platformProfileFor?: ( spec: DeploymentSpec, ) => ExternalPlatformProfile; @@ -358,6 +364,8 @@ export class WorkersForPlatformsBackend implements ProvisioningBackend { readonly hostRoutingKvId: string; readonly auditQueueName?: string; readonly maintenanceRequestTimeoutMs?: number; + /** Stamps `observedAt` on an attestation. Injected so it can be pinned. */ + readonly clock?: () => number; readonly platformProfileFor?: ( spec: DeploymentSpec, ) => ExternalPlatformProfile; @@ -388,6 +396,7 @@ export class WorkersForPlatformsBackend implements ProvisioningBackend { 'maintenanceRequestTimeoutMs must be a positive integer that fits the one-minute capability lifetime', ); } + this.#clock = options.clock ?? Date.now; this.#platformProfileFor = options.platformProfileFor; if ( !options.namespacedState?.dispatchNamespace || @@ -487,6 +496,7 @@ export class WorkersForPlatformsBackend implements ProvisioningBackend { database: DatabaseReference, tenantTag: string, fence: ExternalMutationFence, + options: SeedDeploymentIdentityOptions, ): Promise { await this.#withMutationFence(fence, () => provisionDeploymentIdentityProtocol( @@ -497,7 +507,10 @@ export class WorkersForPlatformsBackend implements ProvisioningBackend { statement.bindings, ), tenantTag, - { caller: 'WorkersForPlatformsBackend.seedDeploymentIdentity' }, + { + caller: 'WorkersForPlatformsBackend.seedDeploymentIdentity', + initialExecutionFenceState: options.initialExecutionFenceState, + }, ), ); } @@ -1822,6 +1835,69 @@ export class WorkersForPlatformsBackend implements ProvisioningBackend { }; } + /** + * Attest the release the hostname actually dispatches to. + * + * The chain starts at the routing entry rather than at + * `releaseScriptName(spec)`, and that ordering is the point: an immutable + * release is only live because the host route names it, so asking the + * expected release whether it exists proves nothing about what is serving. + * When the route names a different release — a rollback that landed, a + * promotion that did not — the attestation reports THAT release, and the + * caller comparing it against its own expectation is what fails closed. + * + * Host routing is eventually consistent, so a caller attesting straight after + * a promotion should go through the shared convergence helper rather than + * treating one stale read as drift. + */ + async attestActiveRoute( + spec: DeploymentSpec, + ): Promise { + const serialized = await this.#client.getHostRouting( + this.#hostRoutingKvId, + spec.routeHostname, + ); + if (serialized === undefined) { + throw new ActiveRouteAttestationError( + `host route '${spec.routeHostname}' dispatches to no release`, + {}, + ); + } + let routed: import('./host-routing.js').HostRoutingTarget; + try { + routed = await parseHostRoutingTarget(serialized); + } catch (cause) { + throw new ActiveRouteAttestationError( + `host route '${spec.routeHostname}' is not a readable dispatch target`, + {}, + { cause }, + ); + } + const live = await this.#inspectDispatchWorker(routed.scriptName); + if (!live) { + throw new ActiveRouteAttestationError( + `host route '${spec.routeHostname}' dispatches to absent release '${routed.scriptName}'`, + { routedScriptName: routed.scriptName }, + ); + } + if (!isSha256(live.desiredSpecDigest)) { + throw new ActiveRouteAttestationError( + `routed release '${routed.scriptName}' carries no fleet specification digest`, + { + routedScriptName: routed.scriptName, + artifactVersion: live.artifactVersion, + }, + ); + } + return { + specDigest: live.desiredSpecDigest, + artifactVersion: live.artifactVersion, + physicalScriptName: routed.scriptName, + source: 'dispatch-route', + observedAt: new Date(this.#clock()).toISOString(), + }; + } + async revokeCredentials( spec: DeploymentSpec, retainedReleases: readonly ExternalReleaseSnapshot[] | undefined, diff --git a/packages/fleet-control/src/wrangler-loop-backend.ts b/packages/fleet-control/src/wrangler-loop-backend.ts index cc2db86..1022981 100644 --- a/packages/fleet-control/src/wrangler-loop-backend.ts +++ b/packages/fleet-control/src/wrangler-loop-backend.ts @@ -10,6 +10,7 @@ import { provisionDeploymentIdentityProtocol, readDeploymentIdentityProtocol, } from '@proofoftech/flowsafe/deployment-identity-protocol'; +import { ActiveRouteAttestationError } from './active-route.js'; import { applicationSecretValues, canonicalApplicationBindings, @@ -22,6 +23,7 @@ import { applyMigrationsWithLedger } from './migration-ledger.js'; import { assertSupportedProviderBindings } from './provider-binding-inventory.js'; import { deploymentSpecDigest } from './spec-digest.js'; import type { + ActiveRouteAttestation, D1Migration, DatabaseExport, DatabaseReference, @@ -35,9 +37,11 @@ import type { ForceDecommissionStep, LiveDeployment, MaintenanceHealth, + PlainWorkerCustomDomain, + PlainWorkerRouteApi, PromotionGuard, ProvisioningBackend, - WorkerZoneRoute, + SeedDeploymentIdentityOptions, } from './types.js'; import { targetDurableObjectTag } from './validation.js'; import type { CommandResult, CommandRunner } from './wrangler-runner.js'; @@ -59,92 +63,6 @@ interface DeploymentStatus { readonly versions: readonly DeploymentVersion[]; } -export interface PlainWorkerCustomDomain { - readonly id: string; - readonly hostname: string; - readonly service: string; -} - -export interface PlainWorkerRouteApi { - withMutationFence( - fence: ExternalMutationFence, - operation: () => Promise, - ): Promise; - queryDatabase( - databaseId: string, - sql: string, - bindings?: readonly string[], - ): Promise>[]>; - batchDatabase( - databaseId: string, - statements: readonly { - readonly sql: string; - readonly bindings?: readonly string[]; - }[], - ): Promise; - getDatabase?(databaseId: string): Promise; - deleteDatabase?(databaseId: string): Promise; - listWorkerDatabaseAttachments(databaseId: string): Promise< - readonly Readonly<{ - scriptName: string; - plane: 'ordinary' | 'dispatch'; - dispatchNamespace?: string; - }>[] - >; - listWorkerR2Attachments?(bucketName: string): Promise< - readonly Readonly<{ - scriptName: string; - plane: 'ordinary' | 'dispatch'; - dispatchNamespace?: string; - }>[] - >; - getR2Bucket?( - bucketName: string, - jurisdiction: import('./types.js').R2Jurisdiction, - ): Promise; - createR2Bucket?( - resource: import('./types.js').ApplicationR2Binding, - fence: ExternalMutationFence, - ): Promise; - assertR2BucketEmpty?( - resource: import('./types.js').ApplicationR2Binding, - ): Promise; - deleteR2Bucket?( - resource: import('./types.js').ApplicationR2Binding, - fence: ExternalMutationFence, - ): Promise; - listCustomDomains(): Promise; - inspectOrdinaryWorkerFootprint(scriptName: string): Promise<{ - readonly scriptPresent: boolean; - readonly workersDevEnabled?: boolean; - readonly previewUrlsEnabled?: boolean; - readonly customDomains: readonly PlainWorkerCustomDomain[]; - readonly zoneRoutes: readonly WorkerZoneRoute[]; - }>; - listDurableObjectNamespaces(scriptName: string): Promise; - listOrdinaryWorkerSecretNames(scriptName: string): Promise; - deleteControlSecrets( - scriptName: string, - secretNames: readonly string[], - fence: ExternalMutationFence, - ): Promise; - attachCustomDomain( - target: { - readonly hostname: string; - readonly service: string; - }, - fence: ExternalMutationFence, - ): Promise; - detachCustomDomain( - domainId: string, - fence: ExternalMutationFence, - ): Promise; - disableOrdinaryWorkerPublicAccess( - scriptName: string, - fence: ExternalMutationFence, - ): Promise; -} - function parseJson(value: string, operation: string): unknown { try { return JSON.parse(value); @@ -309,6 +227,7 @@ export class WranglerLoopBackend implements ProvisioningBackend { readonly #exportDirectory: string; readonly #exportStore: DurableDatabaseExportStore; readonly #maintenanceRequestTimeoutMs: number; + readonly #clock: () => number; constructor(options: { readonly runner: CommandRunner; @@ -317,6 +236,8 @@ export class WranglerLoopBackend implements ProvisioningBackend { readonly exportStore: DurableDatabaseExportStore; readonly fetch?: typeof fetch; readonly maintenanceRequestTimeoutMs?: number; + /** Stamps `observedAt` on an attestation. Injected so it can be pinned. */ + readonly clock?: () => number; }) { if (!options.exportDirectory) throw new Error('exportDirectory is required'); @@ -336,6 +257,7 @@ export class WranglerLoopBackend implements ProvisioningBackend { this.#exportDirectory = resolve(options.exportDirectory); this.#exportStore = options.exportStore; this.#maintenanceRequestTimeoutMs = maintenanceRequestTimeoutMs; + this.#clock = options.clock ?? Date.now; } async #assertMutationFence(fence: ExternalMutationFence): Promise { @@ -459,12 +381,16 @@ export class WranglerLoopBackend implements ProvisioningBackend { database: DatabaseReference, tenantTag: string, fence: ExternalMutationFence, + options: SeedDeploymentIdentityOptions, ): Promise { await provisionDeploymentIdentityProtocol( (statement) => this.#query(database, statement.sql, fence, statement.bindings), tenantTag, - { caller: 'WranglerLoopBackend.seedDeploymentIdentity' }, + { + caller: 'WranglerLoopBackend.seedDeploymentIdentity', + initialExecutionFenceState: options.initialExecutionFenceState, + }, ); } @@ -1797,6 +1723,59 @@ export class WranglerLoopBackend implements ProvisioningBackend { }; } + /** + * Attest the version serving traffic, which for an ordinary Worker is the + * one the deployment object holds at 100%. + * + * `inspect()` cannot answer this. Given an expected artifact version it pins + * that candidate even while the candidate sits at 0%, because a converge has + * to compare a staged upload against the specification before promoting it. + * Reusing it here would report an unpromoted candidate as though it were + * live, which is the exact failure this method exists to make impossible. + * + * The read goes through the provider API rather than the wrangler CLI: the + * CLI is outside the shared rate coordinator, so a poll loop driven through + * it would spend account-wide provider quota that nothing is counting. + * + * `physicalScriptName` is the spec's script rather than a value read back + * from the custom domain, because the hostname-to-script binding is already + * enforced on the path that can change it: `promoteWorker` fails unless the + * custom domain attests this exact script after every promotion, and + * `#attestPromotionRoute` refuses a hostname owned by a Worker outside the + * promotion guard. Re-reading the domain here would spend a third provider + * call against the two-read budget this method documents and learn nothing + * those two checks have not already established. + */ + async attestActiveRoute( + spec: DeploymentSpec, + ): Promise { + const active = await this.#routeApi.inspectActiveWorkerRoute( + spec.scriptName, + ); + if (!active) { + throw new ActiveRouteAttestationError( + `Worker '${spec.scriptName}' has no deployment serving traffic`, + {}, + ); + } + if (!active.specDigest || !isSha256(active.specDigest)) { + throw new ActiveRouteAttestationError( + `routed version '${active.artifactVersion}' of Worker '${spec.scriptName}' carries no fleet specification digest`, + { + routedScriptName: spec.scriptName, + artifactVersion: active.artifactVersion, + }, + ); + } + return { + specDigest: active.specDigest, + artifactVersion: active.artifactVersion, + physicalScriptName: spec.scriptName, + source: 'workers-deployments', + observedAt: new Date(this.#clock()).toISOString(), + }; + } + async revokeCredentials( spec: DeploymentSpec, retainedReleases: readonly ExternalReleaseSnapshot[] | undefined, diff --git a/packages/fleet-control/test/active-route.test.ts b/packages/fleet-control/test/active-route.test.ts new file mode 100644 index 0000000..19b3e6f --- /dev/null +++ b/packages/fleet-control/test/active-route.test.ts @@ -0,0 +1,494 @@ +// SPDX-License-Identifier: Apache-2.0 + +import { describe, expect, it } from 'vitest'; +import { + ActiveRouteAttestationError, + attestConvergedActiveRoute, + attestFleetRecordActiveRoute, +} from '../src/active-route.js'; +import type { + ActiveRouteAttestation, + DeploymentSpec, + FleetRecord, + ProvisioningBackend, +} from '../src/types.js'; + +const TARGET_DIGEST = 'a'.repeat(64); +const PRIOR_DIGEST = 'b'.repeat(64); +const ATTESTED_AT = '2026-08-11T00:00:00.000Z'; + +const spec: DeploymentSpec = { + tenantTag: 'acme', + environment: 'production', + scriptName: 'acme-production', + databaseName: 'acme-production', + compatibilityDate: '2026-08-10', + mainModule: 'worker.js', + modules: [{ name: 'worker.js', content: 'export default {}' }], + authoredBy: 'external', + schemaVersion: 1, + migrations: [], + durableObjectMigrations: [], + durableObjectBindings: [], + maintenanceBaseUrl: 'https://control-acme.example.test', + routeHostname: 'acme.example.test', +}; + +function attestation( + overrides: Partial = {}, +): ActiveRouteAttestation { + return { + specDigest: TARGET_DIGEST, + artifactVersion: 'etag-target', + physicalScriptName: 'acme-production-target', + source: 'dispatch-route', + observedAt: ATTESTED_AT, + ...overrides, + }; +} + +/** + * One answer per attestation call, consumed in order; the last answer repeats + * once the script runs out, which is what a permanently stuck route looks like. + * Only `attestActiveRoute` is implemented — the helper must never reach for + * anything else on the backend, and a thrown "unused" says so if it does. + */ +class ScriptedBackend + implements Pick +{ + readonly reads: string[] = []; + #index = 0; + readonly #answers: readonly (ActiveRouteAttestation | Error)[]; + + constructor(answers: readonly (ActiveRouteAttestation | Error)[]) { + if (answers.length === 0) throw new Error('scripted backend needs answers'); + this.#answers = answers; + } + + async attestActiveRoute( + deployment: DeploymentSpec, + ): Promise { + this.reads.push(deployment.tenantTag); + const answer = + this.#answers[Math.min(this.#index, this.#answers.length - 1)]; + this.#index += 1; + if (answer instanceof Error) throw answer; + return answer as ActiveRouteAttestation; + } +} + +function backendOf(answers: readonly (ActiveRouteAttestation | Error)[]): { + backend: ProvisioningBackend; + scripted: ScriptedBackend; +} { + const scripted = new ScriptedBackend(answers); + return { backend: scripted as unknown as ProvisioningBackend, scripted }; +} + +/** A clock the injected sleep advances, so backoff costs no real time. */ +function fakeTimeline(): { + clock: () => number; + sleep: (delayMs: number) => Promise; + slept: number[]; +} { + let now = 1_000_000; + const slept: number[] = []; + return { + clock: () => now, + sleep: async (delayMs: number) => { + slept.push(delayMs); + now += delayMs; + }, + slept, + }; +} + +function record(overrides: Partial = {}): FleetRecord { + return { + tenantTag: spec.tenantTag, + backend: 'workers-for-platforms', + environment: spec.environment, + scriptName: spec.scriptName, + databaseId: 'db-acme', + databaseName: spec.databaseName, + schemaVersion: spec.schemaVersion, + artifactVersion: 'etag-target', + desiredSpecDigest: TARGET_DIGEST, + durableObjectBindings: [], + routeHostname: spec.routeHostname, + phase: 'ready', + updatedAt: ATTESTED_AT, + ...overrides, + }; +} + +describe('attestConvergedActiveRoute', () => { + it('costs one attestation when the route already matches', async () => { + // #given a route already serving the expected release + const { backend, scripted } = backendOf([attestation()]); + const timeline = fakeTimeline(); + + // #when it is attested + const converged = await attestConvergedActiveRoute( + backend, + spec, + { specDigest: TARGET_DIGEST, artifactVersion: 'etag-target' }, + timeline, + ); + + // #then the provider is read once and nothing is waited on + expect(converged).toEqual(attestation()); + expect(scripted.reads).toHaveLength(1); + expect(timeline.slept).toEqual([]); + }); + + it('waits out a stale route and returns the converged attestation', async () => { + // #given host routing still answering with the release just replaced + const stale = attestation({ + specDigest: PRIOR_DIGEST, + artifactVersion: 'etag-prior', + physicalScriptName: 'acme-production-prior', + }); + const { backend, scripted } = backendOf([stale, stale, attestation()]); + const timeline = fakeTimeline(); + + // #when the convergence helper attests + const converged = await attestConvergedActiveRoute( + backend, + spec, + { specDigest: TARGET_DIGEST, artifactVersion: 'etag-target' }, + timeline, + ); + + // #then it returns the settled attestation after a widening backoff + expect(converged).toEqual(attestation()); + expect(scripted.reads).toHaveLength(3); + expect(timeline.slept).toEqual([1_000, 2_000]); + }); + + it('waits out a route mapping that has not landed yet', async () => { + // #given a key-value route write that is not visible on first read + const { backend, scripted } = backendOf([ + new ActiveRouteAttestationError('dispatches to no release', {}), + attestation(), + ]); + const timeline = fakeTimeline(); + + // #when the convergence helper attests + const converged = await attestConvergedActiveRoute( + backend, + spec, + { specDigest: TARGET_DIGEST, artifactVersion: 'etag-target' }, + timeline, + ); + + // #then the unconverged read is a wait, not a verdict + expect(converged).toEqual(attestation()); + expect(scripted.reads).toHaveLength(2); + }); + + it('fails closed on a permanent mismatch inside a bounded budget', async () => { + // #given a route that never moves off the prior release + const stale = attestation({ + specDigest: PRIOR_DIGEST, + artifactVersion: 'etag-prior', + physicalScriptName: 'acme-production-prior', + }); + const { backend, scripted } = backendOf([stale]); + const timeline = fakeTimeline(); + + // #when the convergence helper attests + const failure = await attestConvergedActiveRoute( + backend, + spec, + { specDigest: TARGET_DIGEST, artifactVersion: 'etag-target' }, + timeline, + ).catch((error: unknown) => error); + + // #then it gives up rather than spinning, and hands back what it last saw + expect(failure).toBeInstanceOf(ActiveRouteAttestationError); + expect((failure as ActiveRouteAttestationError).attestation).toEqual(stale); + expect((failure as ActiveRouteAttestationError).observed).toEqual({ + routedScriptName: 'acme-production-prior', + artifactVersion: 'etag-prior', + specDigest: PRIOR_DIGEST, + }); + expect((failure as ActiveRouteAttestationError).message).toContain( + "still routes artifact 'etag-prior' of 'acme-production-prior'", + ); + expect(scripted.reads).toHaveLength(10); + expect(timeline.slept).toEqual([ + 1_000, 2_000, 4_000, 8_000, 8_000, 8_000, 8_000, 8_000, 8_000, + ]); + expect(timeline.slept.reduce((total, delay) => total + delay, 0)).toBe( + 55_000, + ); + }); + + it('bounds itself even when the injected clock never advances', async () => { + // #given a clock frozen where the elapsed-time bound can never fire + const stale = attestation({ specDigest: PRIOR_DIGEST }); + const { backend, scripted } = backendOf([stale]); + + // #when the convergence helper attests against it + const failure = await attestConvergedActiveRoute( + backend, + spec, + { specDigest: TARGET_DIGEST, artifactVersion: 'etag-target' }, + { clock: () => 1_000_000, sleep: async () => {} }, + ).catch((error: unknown) => error); + + // #then the schedule alone still terminates it + expect(failure).toBeInstanceOf(ActiveRouteAttestationError); + expect(scripted.reads).toHaveLength(10); + }); + + it('fails closed carrying the last refusal when none ever succeeds', async () => { + // #given a hostname that never routes anywhere + const observed = { routedScriptName: 'acme-production-dispatch' }; + const unrouted = new ActiveRouteAttestationError( + 'dispatches to no release', + observed, + ); + const { backend } = backendOf([unrouted]); + const timeline = fakeTimeline(); + + // #when the convergence helper attests + const failure = await attestConvergedActiveRoute( + backend, + spec, + { specDigest: TARGET_DIGEST, artifactVersion: 'etag-target' }, + timeline, + ).catch((error: unknown) => error); + + // #then the refusal that stood is the cause, not something invented + expect(failure).toBeInstanceOf(ActiveRouteAttestationError); + expect( + (failure as ActiveRouteAttestationError).attestation, + ).toBeUndefined(); + expect((failure as ActiveRouteAttestationError).observed).toBe(observed); + expect((failure as ActiveRouteAttestationError).cause).toBe(unrouted); + expect((failure as ActiveRouteAttestationError).message).toContain( + 'dispatches to no release', + ); + expect((failure as ActiveRouteAttestationError).message).not.toContain( + 'still routes artifact', + ); + }); + + it('reports the last different release after earlier refusals', async () => { + // #given initial reads that cannot attest, followed by a routed old release + const refusal = new ActiveRouteAttestationError( + 'dispatches to no release', + {}, + ); + const stale = attestation({ + specDigest: PRIOR_DIGEST, + artifactVersion: 'etag-prior', + physicalScriptName: 'acme-production-prior', + }); + const { backend } = backendOf([refusal, refusal, stale]); + + // #when the route never reaches the expected release within the budget + const failure = await attestConvergedActiveRoute( + backend, + spec, + { specDigest: TARGET_DIGEST, artifactVersion: 'etag-target' }, + fakeTimeline(), + ).catch((error: unknown) => error); + + // #then the final complete observation determines the message shape + expect(failure).toBeInstanceOf(ActiveRouteAttestationError); + expect((failure as ActiveRouteAttestationError).message).toContain( + "still routes artifact 'etag-prior' of 'acme-production-prior'", + ); + expect((failure as ActiveRouteAttestationError).message).not.toContain( + 'dispatches to no release', + ); + expect((failure as ActiveRouteAttestationError).attestation).toBe(stale); + }); + + it('reports the last refusal after an earlier stale release', async () => { + // #given a stale release followed by a route that remains unavailable + const stale = attestation({ + specDigest: PRIOR_DIGEST, + artifactVersion: 'etag-prior', + physicalScriptName: 'acme-production-prior', + }); + const observed = { routedScriptName: 'acme-production-dispatch' }; + const refusal = new ActiveRouteAttestationError( + 'dispatches to no release', + observed, + ); + const { backend } = backendOf([stale, refusal]); + + // #when refusals occupy the rest of the convergence budget + const failure = await attestConvergedActiveRoute( + backend, + spec, + { specDigest: TARGET_DIGEST, artifactVersion: 'etag-target' }, + fakeTimeline(), + ).catch((error: unknown) => error); + + // #then the final refusal replaces the stale complete observation + expect(failure).toBeInstanceOf(ActiveRouteAttestationError); + expect((failure as ActiveRouteAttestationError).cause).toBe(refusal); + expect( + (failure as ActiveRouteAttestationError).attestation, + ).toBeUndefined(); + expect((failure as ActiveRouteAttestationError).observed).toBe(observed); + expect((failure as ActiveRouteAttestationError).message).toContain( + 'dispatches to no release', + ); + expect((failure as ActiveRouteAttestationError).message).not.toContain( + 'etag-prior', + ); + }); + + it('never retries a failure that is not an attestation refusal', async () => { + // #given a transport failure rather than an unconverged route + const transport = new Error('provider connection reset'); + const { backend, scripted } = backendOf([transport]); + const timeline = fakeTimeline(); + + // #when the convergence helper attests + const failure = await attestConvergedActiveRoute( + backend, + spec, + { specDigest: TARGET_DIGEST, artifactVersion: 'etag-target' }, + timeline, + ).catch((error: unknown) => error); + + // #then it propagates immediately instead of spending the budget on it + expect(failure).toBe(transport); + expect(scripted.reads).toHaveLength(1); + expect(timeline.slept).toEqual([]); + }); + + it('matches on the digest alone while the artifact version is pending', async () => { + // #given a first release, whose artifact version is not yet known + const first = attestation({ artifactVersion: 'etag-first-release' }); + const { backend } = backendOf([first]); + const timeline = fakeTimeline(); + + // #when the pending sentinel stands in for an artifact expectation + const converged = await attestConvergedActiveRoute( + backend, + spec, + { specDigest: TARGET_DIGEST, artifactVersion: 'pending' }, + timeline, + ); + + // #then the digest decides the match and the attested version comes back + expect(converged.artifactVersion).toBe('etag-first-release'); + expect(timeline.slept).toEqual([]); + }); + + it('holds the digest even when the artifact version is pending', async () => { + // #given a route serving a release with a different specification + const other = attestation({ specDigest: PRIOR_DIGEST }); + const { backend } = backendOf([other]); + const timeline = fakeTimeline(); + + // #when the pending sentinel stands in for an artifact expectation + const failure = await attestConvergedActiveRoute( + backend, + spec, + { specDigest: TARGET_DIGEST, artifactVersion: 'pending' }, + timeline, + ).catch((error: unknown) => error); + + // #then the sentinel waives the version check only, never the digest + expect(failure).toBeInstanceOf(ActiveRouteAttestationError); + }); + + it('refuses a backoff schedule it cannot honour', async () => { + // #given a maximum delay below the delay the schedule starts at + const { backend } = backendOf([attestation()]); + + // #when the convergence helper is configured with it + const failure = attestConvergedActiveRoute( + backend, + spec, + { specDigest: TARGET_DIGEST, artifactVersion: 'etag-target' }, + { initialRetryDelayMs: 4_000, maxRetryDelayMs: 1_000 }, + ); + + // #then it refuses at configuration time rather than backing off downward + await expect(failure).rejects.toThrow(/must not be below/); + }); +}); + +describe('attestFleetRecordActiveRoute', () => { + it('reports agreement with the desired release', async () => { + // #given a record whose desired release is the one being served + const { backend, scripted } = backendOf([attestation()]); + + // #when the record's active route is read + const read = await attestFleetRecordActiveRoute({ + record: record(), + backend, + spec, + }); + + // #then it agrees, on exactly one provider round trip + expect(read).toEqual({ attestation: attestation(), matchesDesired: true }); + expect(scripted.reads).toHaveLength(1); + }); + + it('reports drift instead of failing on it', async () => { + // #given a record desiring a release the route no longer serves + const drifted = attestation({ + specDigest: PRIOR_DIGEST, + artifactVersion: 'etag-prior', + physicalScriptName: 'acme-production-prior', + }); + const { backend, scripted } = backendOf([drifted, drifted]); + + // #when the record's active route is read + const read = await attestFleetRecordActiveRoute({ + record: record(), + backend, + spec, + }); + + // #then the disagreement comes back as data, with no retry spent on it + expect(read).toEqual({ attestation: drifted, matchesDesired: false }); + expect(scripted.reads).toHaveLength(1); + }); + + it('treats a pending artifact version as no version expectation', async () => { + // #given a record persisted before its first release settled + const first = attestation({ artifactVersion: 'etag-first-release' }); + const { backend } = backendOf([first]); + + // #when the record's active route is read + const read = await attestFleetRecordActiveRoute({ + record: record({ artifactVersion: 'pending' }), + backend, + spec, + }); + + // #then the digest alone settles the comparison + expect(read.matchesDesired).toBe(true); + }); + + it('lets an unattestable route fail rather than reporting it as drift', async () => { + // #given a hostname that routes nowhere + const unrouted = new ActiveRouteAttestationError( + 'dispatches to no release', + {}, + ); + const { backend } = backendOf([unrouted]); + + // #when the record's active route is read + const failure = await attestFleetRecordActiveRoute({ + record: record(), + backend, + spec, + }).catch((error: unknown) => error); + + // #then it raises: "cannot tell" is not the same answer as "does not match" + expect(failure).toBe(unrouted); + }); +}); diff --git a/packages/fleet-control/test/backend-switch.test.ts b/packages/fleet-control/test/backend-switch.test.ts index b1388a7..cb0f895 100644 --- a/packages/fleet-control/test/backend-switch.test.ts +++ b/packages/fleet-control/test/backend-switch.test.ts @@ -1025,6 +1025,7 @@ describe('backend switch state machine', () => { } as unknown as ProvisioningBackend; const result = await provisionDeployment({ + initialExecutionFenceState: 'open', backend, store, spec: targetSpec, diff --git a/packages/fleet-control/test/cloudflare-client.test.ts b/packages/fleet-control/test/cloudflare-client.test.ts index bd8f407..59da20e 100644 --- a/packages/fleet-control/test/cloudflare-client.test.ts +++ b/packages/fleet-control/test/cloudflare-client.test.ts @@ -1,6 +1,7 @@ // SPDX-License-Identifier: Apache-2.0 import { describe, expect, it, vi } from 'vitest'; +import { ActiveRouteAttestationError } from '../src/active-route.js'; import { CloudflareProvisioningClient, type DurableDatabaseExportStore, @@ -2800,4 +2801,136 @@ describe('CloudflareProvisioningClient', () => { expect(mutations).toEqual(['attach', 'detach']); expect(fence.assertOwned).toHaveBeenCalledTimes(2); }); + + it('reduces an active route to one version at 100% or refuses', async () => { + // #given a Worker whose deployment traffic split is controlled per case + let versions: readonly Readonly<{ + percentage: number; + version_id: string; + }>[] = [{ percentage: 100, version_id: 'version-live' }]; + let bindings: readonly Readonly>[] = [ + { type: 'plain_text', name: 'FLEET_SPEC_DIGEST', text: 'a'.repeat(64) }, + ]; + let scriptPresent = true; + let versionReads = 0; + const client = activeRouteClient(() => ({ + versions, + bindings, + scriptPresent, + onVersionRead: () => { + versionReads += 1; + }, + })); + + // #when a single version holds all of the traffic + // #then it is attested with the digest its bindings carry + await expect( + client.inspectActiveWorkerRoute('acme-production'), + ).resolves.toEqual({ + artifactVersion: 'version-live', + specDigest: 'a'.repeat(64), + }); + + // #when a candidate is staged into the deployment at 0% + versions = [ + { percentage: 100, version_id: 'version-live' }, + { percentage: 0, version_id: 'version-candidate' }, + ]; + const staged = await client + .inspectActiveWorkerRoute('acme-production') + .catch((error: unknown) => error); + + // #then the deployment is refused rather than reduced by picking a winner, + // and the staged candidate is never what comes back + expect(staged).toBeInstanceOf(ActiveRouteAttestationError); + expect((staged as ActiveRouteAttestationError).message).toContain( + 'exactly one current version receiving 100% of traffic', + ); + expect((staged as ActiveRouteAttestationError).observed).toEqual({ + routedScriptName: 'acme-production', + trafficSplit: [ + { artifactVersion: 'version-live', percentage: 100 }, + { artifactVersion: 'version-candidate', percentage: 0 }, + ], + }); + + // #when traffic is genuinely split + versions = [ + { percentage: 60, version_id: 'version-old' }, + { percentage: 40, version_id: 'version-new' }, + ]; + const split = await client + .inspectActiveWorkerRoute('acme-production') + .catch((error: unknown) => error); + + // #then the refusal carries the percentages, and the larger share is not + // promoted to "the routed version" + expect((split as ActiveRouteAttestationError).observed).toEqual({ + routedScriptName: 'acme-production', + trafficSplit: [ + { artifactVersion: 'version-old', percentage: 60 }, + { artifactVersion: 'version-new', percentage: 40 }, + ], + }); + + // #when the routed version carries no fleet digest binding + versions = [{ percentage: 100, version_id: 'version-live' }]; + bindings = [{ type: 'plain_text', name: 'FLEET_ENVIRONMENT', text: 'p' }]; + + // #then the version is still reported, with the absence stated as absence + await expect( + client.inspectActiveWorkerRoute('acme-production'), + ).resolves.toEqual({ + artifactVersion: 'version-live', + specDigest: undefined, + }); + + // #when the script does not exist + scriptPresent = false; + const reads = versionReads; + + // #then there is nothing to attest and no version read is spent + await expect( + client.inspectActiveWorkerRoute('acme-production'), + ).resolves.toBeUndefined(); + expect(versionReads).toBe(reads); + }); }); + +function activeRouteClient( + state: () => { + readonly versions: readonly Readonly<{ + percentage: number; + version_id: string; + }>[]; + readonly bindings: readonly Readonly>[]; + readonly scriptPresent: boolean; + readonly onVersionRead: () => void; + }, +) { + return new CloudflareProvisioningClient({ + accountId: 'account', + apiToken: 'token', + rateCoordinator: testRateCoordinator(), + dispatchNamespace: 'fleet', + fetch: async (input) => { + const url = new URL( + typeof input === 'string' + ? input + : input instanceof URL + ? input.href + : input.url, + ); + const current = state(); + if (url.pathname.endsWith('/workers/scripts/acme-production/deployments')) + return current.scriptPresent + ? envelope({ deployments: [{ versions: current.versions }] }) + : new Response(null, { status: 404 }); + if (url.pathname.includes('/workers/scripts/acme-production/versions/')) { + current.onVersionRead(); + return envelope({ resources: { bindings: current.bindings } }); + } + throw new Error(`unexpected Cloudflare request: ${url.href}`); + }, + }); +} diff --git a/packages/fleet-control/test/fleet.test.ts b/packages/fleet-control/test/fleet.test.ts index 4e1b560..6ac0060 100644 --- a/packages/fleet-control/test/fleet.test.ts +++ b/packages/fleet-control/test/fleet.test.ts @@ -2,6 +2,7 @@ import { createHash } from 'node:crypto'; import { describe, expect, it } from 'vitest'; +import { ActiveRouteAttestationError } from '../src/active-route.js'; import { canonicalApplicationBindings } from '../src/application-bindings.js'; import type { BridgeMutationPlan, @@ -26,12 +27,18 @@ import { } from '../src/platform-resources.js'; import { providerBindingIdentitiesForInspection } from '../src/provider-binding-inventory.js'; import { provisionDeployment } from '../src/provision.js'; +import { fleetSettlementKey } from '../src/settlement.js'; import { deploymentSpecDigest } from '../src/spec-digest.js'; import type { + ActiveRouteAttestation, DatabaseReference, DeploymentSpec, + ExternalMutationFence, + ExternalReleaseSnapshot, FleetRecord, FleetResourceInventory, + FleetSettlementContext, + FleetSettlementHost, FleetStateLease, FleetStateStore, LiveDeployment, @@ -39,6 +46,7 @@ import type { PromotionGuard, ProvisioningBackend, ProvisioningBackendKind, + SeedDeploymentIdentityOptions, } from '../src/types.js'; import { externalReleaseScriptName } from '../src/workers-for-platforms-backend.js'; @@ -143,6 +151,9 @@ function spec(item: FleetRecord, schemaVersion = 2): DeploymentSpec { }; } +/** Pinned so an attestation these fakes return is comparable by value. */ +const ATTESTED_AT = '2026-08-11T00:00:00.000Z'; + const healthy: MaintenanceHealth = { armed: true, nextAlarmAt: 11_000, @@ -155,6 +166,10 @@ class FleetBackend implements ProvisioningBackend { readonly immutableExternalArtifacts?: true; readonly calls: string[] = []; readonly live = new Map(); + /** What the route names per tenant, written by promotion. */ + readonly routed = new Map(); + /** Strands the route on something else, to drive a settlement mismatch. */ + routeDrift: ActiveRouteAttestation | undefined; failTenant: string | undefined; inspectFailureTenant: string | undefined; maintenanceFailureTenant: string | undefined; @@ -186,11 +201,18 @@ class FleetBackend implements ProvisioningBackend { throw new Error('unused'); } + // Declares the fence state so it appears in the call log. A fake that simply + // omitted the options argument would still satisfy the interface, and an + // unthreaded fence state would be invisible to every assertion below. async seedDeploymentIdentity( _database: DatabaseReference, tenantTag: string, + _fence: ExternalMutationFence, + options: SeedDeploymentIdentityOptions, ): Promise { - this.calls.push(`identity:${tenantTag}`); + this.calls.push( + `identity:${tenantTag}:${options.initialExecutionFenceState}`, + ); } async readDeploymentIdentity( @@ -343,11 +365,27 @@ class FleetBackend implements ProvisioningBackend { return { artifactVersion: `v${deployment.schemaVersion}`, created: false }; } + // Promotion is what makes a release routed, here as in the provider, so the + // attestation below reads what promotion published rather than a value the + // fake was handed separately. async promoteWorker( deployment: DeploymentSpec, _guard: PromotionGuard, + _outboundPolicy?: FleetRecord['outboundPolicy'], + _fence?: unknown, + expectedArtifactVersion?: string, ): Promise { this.calls.push(`promote:${deployment.tenantTag}`); + this.routed.set(deployment.tenantTag, { + specDigest: deploymentSpecDigest(deployment), + artifactVersion: + expectedArtifactVersion ?? + this.live.get(deployment.tenantTag)?.artifactVersion ?? + `v${deployment.schemaVersion}`, + physicalScriptName: deployment.scriptName, + source: 'workers-deployments', + observedAt: ATTESTED_AT, + }); } async ensureMaintenance( @@ -373,6 +411,21 @@ class FleetBackend implements ProvisioningBackend { return completeLiveDeployment(live); } + async attestActiveRoute( + deployment: DeploymentSpec, + ): Promise { + this.calls.push(`attest:${deployment.tenantTag}`); + const attestation = + this.routeDrift ?? this.routed.get(deployment.tenantTag); + if (!attestation) { + throw new ActiveRouteAttestationError( + `no deployment serves '${deployment.routeHostname}'`, + {}, + ); + } + return attestation; + } + async revokeCredentials(): Promise { throw new Error('unused'); } @@ -408,6 +461,8 @@ class FleetBackend implements ProvisioningBackend { class FleetStore implements FleetStateStore { readonly records = new Map(); readonly leases = new Set(); + /** Every accepted write, so a test can place one relative to a callback. */ + readonly puts: FleetRecord[] = []; failNextReadyPut = false; failNextMigratingSchemaPut = false; failNextPutWhen: @@ -470,6 +525,7 @@ class FleetStore implements FleetStateStore { throw new Error('state write failed after route promotion'); } this.records.set(value.tenantTag, value); + this.puts.push(value); } async delete(tenantTag: string, _environment?: string): Promise { @@ -493,6 +549,9 @@ class ImmutableFleetBackend extends FleetBackend { override readonly immutableExternalArtifacts = true as const; readonly releases = new Map(); routedScriptName: string | undefined; + /** Answers this attestation before the real route, N reads long. */ + staleRoute: ActiveRouteAttestation | undefined; + staleRouteReads = 0; invalidateCandidate = false; failNextRetirement = false; readonly retiredScriptNames: string[] = []; @@ -577,6 +636,41 @@ class ImmutableFleetBackend extends FleetBackend { this.lastPromotedPolicyHosts = outboundPolicy?.policyHosts; } + // Attests the release the ROUTE names, not the one the spec expects, so a + // test can put the two out of step and see the attestation say so. + override async attestActiveRoute( + deployment: DeploymentSpec, + ): Promise { + this.calls.push(`attest:${deployment.tenantTag}`); + if (this.routeDrift) return this.routeDrift; + if (this.staleRouteReads > 0) { + this.staleRouteReads -= 1; + const stale = this.staleRoute; + if (stale) return stale; + } + const routed = this.routedScriptName; + if (routed === undefined) { + throw new ActiveRouteAttestationError( + `host route '${deployment.routeHostname}' dispatches to no release`, + {}, + ); + } + const release = this.releases.get(routed); + if (!release) { + throw new ActiveRouteAttestationError( + `host route '${deployment.routeHostname}' dispatches to absent release '${routed}'`, + { routedScriptName: routed }, + ); + } + return { + specDigest: release.desiredSpecDigest, + artifactVersion: release.artifactVersion, + physicalScriptName: routed, + source: 'dispatch-route', + observedAt: ATTESTED_AT, + }; + } + async deleteRetainedRelease( _deployment: DeploymentSpec, release: import('../src/types.js').ExternalReleaseSnapshot, @@ -3089,6 +3183,7 @@ describe('fleet operations', () => { }, }); + const rollbackHost = new RecordingSettlementHost(store); const rollback = () => rollbackExternalRelease({ store, @@ -3100,6 +3195,7 @@ describe('fleet operations', () => { maintenanceAdmin: 'maintenance-admin-secret-value-00001', application: { API_ONE: initialSecret }, }, + settlement: rollbackHost, }); const persistedRollback = backend.releases.get(activePhysicalScriptName); if (!persistedRollback) @@ -3116,7 +3212,27 @@ describe('fleet operations', () => { await expect(rollback()).rejects.toThrow(/state write failed/); expect(backend.routedScriptName).toBe(activePhysicalScriptName); expect((await store.get('acme', 'production'))?.phase).toBe('rolling-back'); + // The settle-succeeded-then-crashed window: traffic moved back and the + // host was told, but the write recording it was lost. + expect(rollbackHost.settlements).toHaveLength(1); + expect( + (await store.get('acme', 'production'))?.settledSettlementKey, + ).toBeUndefined(); const rolledBack = await rollback(); + // #then the re-entry re-delivers the SAME settlement, and still names the + // abandoned release as `prior` rather than the one it replaced. + expect(rollbackHost.settlements).toHaveLength(2); + expect(rollbackHost.settlements[1]?.settlementKey).toBe( + rollbackHost.settlements[0]?.settlementKey, + ); + expect( + rollbackHost.settlements.map((context) => context.alreadySettled), + ).toEqual([false, false]); + expect( + rollbackHost.settlements.map( + (context) => context.prior?.physicalScriptName, + ), + ).toEqual([targetPhysicalScriptName, targetPhysicalScriptName]); expect(backend.routedScriptName).toBe(activePhysicalScriptName); expect(backend.lastPromotedPolicyHosts).toEqual(['api.example.com']); expect(rolledBack).toMatchObject({ @@ -3134,6 +3250,20 @@ describe('fleet operations', () => { releaseSchemaVersion: 2, }, }); + // `prior` on a reversal is the release traffic just LEFT, not the one it + // replaced: a host undoing its own effects has to undo the right ones. + expect(rollbackHost.settlements.at(-1)).toMatchObject({ + entry: 'rollback', + target: { physicalScriptName: activePhysicalScriptName }, + prior: { + physicalScriptName: targetPhysicalScriptName, + specDigest: deploymentSpecDigest(targetSpec), + }, + }); + expect(rollbackHost.leaseHeld.at(-1)).toBe(true); + expect(rolledBack.settledSettlementKey).toBe( + rollbackHost.settlements.at(-1)?.settlementKey, + ); const deployCallsBeforeConvergence = backend.calls.filter((call) => call.startsWith('deploy:'), @@ -3164,6 +3294,7 @@ describe('fleet operations', () => { await expect( provisionDeployment({ + initialExecutionFenceState: 'open', backend, store, spec: initialSpec, @@ -3580,6 +3711,13 @@ describe('fleet operations', () => { expect(migrationIndex).toBeGreaterThanOrEqual(0); expect(stateIndex).toBeGreaterThan(migrationIndex); expect(deployIndex).toBeGreaterThan(stateIndex); + // A migration re-stamps a database this deployment already owns, and asks + // for 'open' — the state a live, executing deployment already has. Seeding + // 'migration-locked' here would stop a running deployment mid-migration, + // which is why migrateFleet takes no fence option at all. + expect( + backend.calls.filter((call) => call.startsWith('identity:')), + ).toEqual(['identity:acme:open']); }); it('returns a stable sorted version report', () => { @@ -3589,3 +3727,497 @@ describe('fleet operations', () => { ]); }); }); + +class RecordingSettlementHost implements FleetSettlementHost { + readonly settlements: FleetSettlementContext[] = []; + readonly leaseHeld: boolean[] = []; + readonly writesBefore: number[] = []; + throwOnce: string | undefined; + readonly #store: FleetStore; + + constructor(store: FleetStore) { + this.#store = store; + } + + async settle(context: FleetSettlementContext): Promise { + this.settlements.push(context); + this.leaseHeld.push( + this.#store.leases.has(`${context.tenantTag}:${context.environment}`), + ); + this.writesBefore.push(this.#store.puts.length); + if (this.throwOnce) { + const message = this.throwOnce; + this.throwOnce = undefined; + throw new Error(message); + } + } +} + +/** A ready immutable deployment already serving the release it desires. */ +function readyImmutableFleet(): { + backend: ImmutableFleetBackend; + current: FleetRecord; + targetSpec: DeploymentSpec; + activeRelease: ExternalReleaseSnapshot; +} { + const base = record('acme'); + const targetSpec = spec(base, 1); + const backend = new ImmutableFleetBackend(); + const platformTarget = backend.describeExternalPlatformTarget(targetSpec); + const activeRelease: ExternalReleaseSnapshot = { + physicalScriptName: externalReleaseScriptName(targetSpec), + specDigest: deploymentSpecDigest(targetSpec), + artifactVersion: 'etag:active', + releaseSchemaVersion: 1, + }; + const current: FleetRecord = { + ...base, + schemaVersion: 1, + desiredSpecDigest: activeRelease.specDigest, + artifactVersion: activeRelease.artifactVersion, + activeRelease, + durableObjectBindings: [], + platformTarget, + outboundPolicy: platformTarget.outboundPolicy, + platformResources: { + maintenanceCapabilityPublicKey: MAINTENANCE_PUBLIC_KEY, + stateWorker: { + scriptName: externalStateScriptName(targetSpec), + artifactVersion: 'state-v1', + artifactDigest: platformTarget.stateArtifactDigest, + durableObjectBindings: [], + namespaceIds: [], + }, + egressProxy: { + scriptName: externalEgressProxyScriptName(targetSpec), + artifactVersion: 'egress-v1', + artifactDigest: platformTarget.egressArtifactDigest, + ...platformTarget.outboundPolicy, + }, + }, + }; + backend.releases.set(activeRelease.physicalScriptName, { + ...liveFor(current), + scriptName: activeRelease.physicalScriptName, + durableObjectBindings: [], + schemaVersion: 1, + }); + backend.routedScriptName = activeRelease.physicalScriptName; + return { backend, current, targetSpec, activeRelease }; +} + +/** A ready immutable deployment whose PLATFORM profile has moved on. */ +function platformOnlyChangeFleet(): { + backend: ImmutableFleetBackend; + current: FleetRecord; + targetSpec: DeploymentSpec; +} { + const base = record('acme'); + const targetSpec = spec(base, 1); + const backend = new ImmutableFleetBackend(); + const initialTarget = { + ...backend.describeExternalPlatformTarget(targetSpec), + d1SchemaVersion: 2, + d1SchemaHistoryDigest: 'f'.repeat(64), + }; + const activeRelease: ExternalReleaseSnapshot = { + physicalScriptName: externalReleaseScriptName(targetSpec), + specDigest: deploymentSpecDigest(targetSpec), + artifactVersion: 'etag:active', + releaseSchemaVersion: 1, + }; + const current: FleetRecord = { + ...base, + schemaVersion: 2, + desiredSpecDigest: activeRelease.specDigest, + artifactVersion: activeRelease.artifactVersion, + activeRelease, + durableObjectBindings: [], + platformTarget: initialTarget, + outboundPolicy: initialTarget.outboundPolicy, + platformResources: { + maintenanceCapabilityPublicKey: MAINTENANCE_PUBLIC_KEY, + stateWorker: { + scriptName: externalStateScriptName(targetSpec), + artifactVersion: 'state-v1', + artifactDigest: initialTarget.stateArtifactDigest, + durableObjectBindings: [], + namespaceIds: [], + }, + egressProxy: { + scriptName: externalEgressProxyScriptName(targetSpec), + artifactVersion: 'egress-v1', + artifactDigest: initialTarget.egressArtifactDigest, + ...initialTarget.outboundPolicy, + }, + }, + }; + backend.releases.set(activeRelease.physicalScriptName, { + ...liveFor(current), + scriptName: activeRelease.physicalScriptName, + durableObjectBindings: [], + schemaVersion: activeRelease.releaseSchemaVersion, + }); + backend.routedScriptName = activeRelease.physicalScriptName; + backend.platformStateDigest = 'd'.repeat(64); + backend.platformPolicyHosts = ['narrow.example.com']; + return { backend, current, targetSpec }; +} + +function platformOnlyMigrate( + store: FleetStore, + backend: ImmutableFleetBackend, + current: FleetRecord, + targetSpec: DeploymentSpec, + host: FleetSettlementHost, +): Promise { + return migrateFleet({ + store, + records: [current], + canaryTenantTags: [], + backendFor: () => backend, + specFor: () => targetSpec, + secretsFor: () => ({ + deploymentIdentity: 'deployment-identity-secret-value-0001', + maintenanceAdmin: 'maintenance-admin-secret-value-00001', + }), + settlementFor: () => host, + }); +} + +describe('lease-held settlement', () => { + it('settles a migration after attestation and before the settling write', async () => { + // #given a plain deployment with a newer specification to migrate to + const acme = record('acme'); + const backend = new FleetBackend(); + const store = new FleetStore(); + await store.put(acme); + const target = spec(acme, 2); + const host = new RecordingSettlementHost(store); + + // #when the fleet migrates with a settling host + const [migrated] = await migrateFleet({ + store, + records: [acme], + canaryTenantTags: [], + backendFor: () => backend, + specFor: () => target, + secretsFor: () => ({ + deploymentIdentity: 'deployment-identity-secret-value-0001', + maintenanceAdmin: 'maintenance-admin-secret-value-00001', + }), + settlementFor: () => host, + }); + + // #then it settled once, while the lease was held, after the route was + // attested and before the write that records the settlement + expect(host.settlements).toHaveLength(1); + expect(host.leaseHeld).toEqual([true]); + expect(backend.calls.indexOf('attest:acme')).toBeGreaterThan( + backend.calls.indexOf('promote:acme'), + ); + expect(host.writesBefore[0]).toBe(store.puts.length - 1); + + const settlementKey = fleetSettlementKey({ + tenantTag: 'acme', + environment: 'production', + specDigest: deploymentSpecDigest(target), + artifactVersion: 'v2', + }); + expect(host.settlements[0]).toMatchObject({ + tenantTag: 'acme', + environment: 'production', + entry: 'migration', + settlementKey, + alreadySettled: false, + attestation: { artifactVersion: 'v2', source: 'workers-deployments' }, + target: { specDigest: deploymentSpecDigest(target) }, + }); + expect(migrated?.settledSettlementKey).toBe(settlementKey); + }); + + it('attests without a settling host, and refuses a route serving something else', async () => { + // #given a converge whose route was left on a foreign release, and no host + const { backend, current, targetSpec } = readyImmutableFleet(); + const store = new FleetStore(); + await store.put(current); + backend.routeDrift = { + specDigest: 'f'.repeat(64), + artifactVersion: 'etag:stranger', + physicalScriptName: 'acme-production-stranger', + source: 'dispatch-route', + observedAt: '2026-08-11T00:00:00.000Z', + }; + + // #when the fleet converges + const converge = migrateFleet({ + store, + records: [current], + canaryTenantTags: [], + backendFor: () => backend, + specFor: () => targetSpec, + secretsFor: () => ({ + deploymentIdentity: 'deployment-identity-secret-value-0001', + maintenanceAdmin: 'maintenance-admin-secret-value-00001', + }), + routeAttestation: { convergenceBudgetMs: 1, initialRetryDelayMs: 1 }, + }); + + // #then attestation ran anyway and failed the converge closed + await expect(converge).rejects.toThrow(/did not converge/); + expect(backend.calls).toContain('attest:acme'); + }); + + it('attests every routine converge but settles only the first', async () => { + // #given a ready deployment already serving what it desires + const { backend, current, targetSpec, activeRelease } = + readyImmutableFleet(); + const store = new FleetStore(); + await store.put(current); + const host = new RecordingSettlementHost(store); + const converge = (record_: FleetRecord) => + migrateFleet({ + store, + records: [record_], + canaryTenantTags: [], + backendFor: () => backend, + specFor: () => targetSpec, + secretsFor: () => ({ + deploymentIdentity: 'deployment-identity-secret-value-0001', + maintenanceAdmin: 'maintenance-admin-secret-value-00001', + }), + settlementFor: () => host, + }); + + // #when the same unchanged deployment is reconciled twice + const [first] = await converge(current); + if (!first) throw new Error('convergence returned no record'); + const writesAfterFirst = store.puts.length; + const [second] = await converge(first); + + // #then the first settled, the second attested and did neither settle nor + // write — a fleet on a reconcile schedule settles once, not forever + expect(host.settlements).toHaveLength(1); + expect(host.settlements[0]).toMatchObject({ + entry: 'ready-convergence', + alreadySettled: false, + target: { artifactVersion: activeRelease.artifactVersion }, + }); + expect(first.settledSettlementKey).toBe(host.settlements[0]?.settlementKey); + expect(second?.settledSettlementKey).toBe(first.settledSettlementKey); + expect(store.puts.length).toBe(writesAfterFirst); + expect(backend.calls.filter((call) => call === 'attest:acme')).toHaveLength( + 2, + ); + }); + + it('waits out a route that has not converged before settling', async () => { + // #given host routing answering once with the release just replaced + const { backend, current, targetSpec } = readyImmutableFleet(); + const store = new FleetStore(); + await store.put(current); + backend.staleRoute = { + specDigest: 'c'.repeat(64), + artifactVersion: 'etag:prior', + physicalScriptName: 'acme-production-prior', + source: 'dispatch-route', + observedAt: '2026-08-11T00:00:00.000Z', + }; + backend.staleRouteReads = 1; + const host = new RecordingSettlementHost(store); + + // #when the fleet converges + const [converged] = await migrateFleet({ + store, + records: [current], + canaryTenantTags: [], + backendFor: () => backend, + specFor: () => targetSpec, + secretsFor: () => ({ + deploymentIdentity: 'deployment-identity-secret-value-0001', + maintenanceAdmin: 'maintenance-admin-secret-value-00001', + }), + settlementFor: () => host, + routeAttestation: { initialRetryDelayMs: 1, maxRetryDelayMs: 1 }, + }); + + // #then the stale read was waited out rather than read as drift + expect(backend.calls.filter((call) => call === 'attest:acme')).toHaveLength( + 2, + ); + expect(host.settlements).toHaveLength(1); + expect(converged?.settledSettlementKey).toBe( + host.settlements[0]?.settlementKey, + ); + }); + + it('leaves a platform-only migration resumable when settlement throws', async () => { + // #given a platform profile change on a ready immutable deployment + const { backend, current, targetSpec } = platformOnlyChangeFleet(); + const store = new FleetStore(); + await store.put(current); + const host = new RecordingSettlementHost(store); + host.throwOnce = 'settlement ledger is unavailable'; + const migrate = () => + platformOnlyMigrate(store, backend, current, targetSpec, host); + + // #when settlement throws, then the migration is re-entered + await expect(migrate()).rejects.toThrow(/settlement ledger is unavailable/); + const stranded = await store.get('acme', 'production'); + const [settled] = await migrate(); + + // #then the throw left the migration where a retry resumes it, and the + // retry delivered the same settlement rather than a new one + expect(stranded).toMatchObject({ + phase: 'migrating', + migrationIntent: { platformOnly: true, subphase: 'route-published' }, + }); + expect(stranded?.settledSettlementKey).toBeUndefined(); + expect(host.settlements).toHaveLength(2); + expect(host.settlements[1]?.settlementKey).toBe( + host.settlements[0]?.settlementKey, + ); + expect(host.settlements.map((context) => context.entry)).toEqual([ + 'platform-only', + 'platform-only', + ]); + expect(settled?.settledSettlementKey).toBe( + host.settlements[0]?.settlementKey, + ); + }); + + it('re-fires a platform-only settlement whose settling write was lost', async () => { + // #given a platform-only migration whose settlement SUCCEEDS and whose + // settling write is then lost — the window a throw cannot reach + const { backend, current, targetSpec } = platformOnlyChangeFleet(); + const store = new FleetStore(); + await store.put(current); + const host = new RecordingSettlementHost(store); + const migrate = () => + platformOnlyMigrate(store, backend, current, targetSpec, host); + store.failNextReadyPut = true; + + // #when the settling write is lost, then the migration is re-entered + await expect(migrate()).rejects.toThrow(/state write failed/); + const stranded = await store.get('acme', 'production'); + const [settled] = await migrate(); + + // #then the lost write left the migration re-enterable, and both deliveries + // carry alreadySettled: false because neither settling write was durable + expect(stranded).toMatchObject({ + phase: 'migrating', + migrationIntent: { platformOnly: true, subphase: 'route-published' }, + }); + expect(stranded?.settledSettlementKey).toBeUndefined(); + expect(host.settlements).toHaveLength(2); + expect(host.settlements[1]?.settlementKey).toBe( + host.settlements[0]?.settlementKey, + ); + expect(host.settlements.map((context) => context.alreadySettled)).toEqual([ + false, + false, + ]); + expect(settled?.settledSettlementKey).toBe( + host.settlements[0]?.settlementKey, + ); + }); + + it('re-fires a migration settlement whose settling write was lost', async () => { + // #given a migration whose settlement succeeds and whose migrated write + // is then lost + const acme = record('acme'); + const backend = new FleetBackend(); + const store = new FleetStore(); + await store.put(acme); + const target = spec(acme, 2); + const host = new RecordingSettlementHost(store); + const migrate = () => + migrateFleet({ + store, + records: [acme], + canaryTenantTags: [], + backendFor: () => backend, + specFor: () => target, + secretsFor: () => ({ + deploymentIdentity: 'deployment-identity-secret-value-0001', + maintenanceAdmin: 'maintenance-admin-secret-value-00001', + }), + settlementFor: () => host, + }); + store.failNextReadyPut = true; + + // #when the settling write is lost, then the migration is re-entered + await expect(migrate()).rejects.toThrow(/state write failed/); + const stranded = await store.get('acme', 'production'); + const [migrated] = await migrate(); + + // #then the same settlement is delivered again rather than a new one, and + // the retry reaches the ready state the lost write was carrying + expect(stranded?.phase).toBe('migrating'); + expect(stranded?.settledSettlementKey).toBeUndefined(); + expect(host.settlements).toHaveLength(2); + expect(host.settlements[1]?.settlementKey).toBe( + host.settlements[0]?.settlementKey, + ); + expect(host.settlements.map((context) => context.alreadySettled)).toEqual([ + false, + false, + ]); + expect(migrated).toMatchObject({ + phase: 'ready', + settledSettlementKey: host.settlements[0]?.settlementKey, + }); + }); + + it('re-fires a convergence settlement whose settling write was lost', async () => { + // #given a ready deployment whose convergence settles and whose added + // settling write is then lost + const { backend, current, targetSpec } = readyImmutableFleet(); + const store = new FleetStore(); + await store.put(current); + const host = new RecordingSettlementHost(store); + const converge = (from: FleetRecord) => + migrateFleet({ + store, + records: [from], + canaryTenantTags: [], + backendFor: () => backend, + specFor: () => targetSpec, + secretsFor: () => ({ + deploymentIdentity: 'deployment-identity-secret-value-0001', + maintenanceAdmin: 'maintenance-admin-secret-value-00001', + }), + settlementFor: () => host, + }); + store.failNextPutWhen = (next) => + next.settledSettlementKey + ? 'state write failed after settlement' + : undefined; + + // #when the settling write is lost, then the deployment is reconciled + // twice more + await expect(converge(current)).rejects.toThrow(/after settlement/); + const stranded = await store.get('acme', 'production'); + const [recovered] = await converge(current); + if (!recovered) throw new Error('convergence returned no record'); + const writesAfterRecovery = store.puts.length; + await converge(recovered); + + // #then the lost write meant the settlement re-fired under the same key, + // and once it was recorded the next reconcile settled nothing and wrote + // nothing at all + expect(stranded?.settledSettlementKey).toBeUndefined(); + expect(host.settlements).toHaveLength(2); + expect(host.settlements[1]?.settlementKey).toBe( + host.settlements[0]?.settlementKey, + ); + expect(host.settlements.map((context) => context.alreadySettled)).toEqual([ + false, + false, + ]); + expect(recovered.settledSettlementKey).toBe( + host.settlements[0]?.settlementKey, + ); + expect(store.puts.length).toBe(writesAfterRecovery); + }); +}); diff --git a/packages/fleet-control/test/provision.test.ts b/packages/fleet-control/test/provision.test.ts index 8b1d8de..76cb55b 100644 --- a/packages/fleet-control/test/provision.test.ts +++ b/packages/fleet-control/test/provision.test.ts @@ -1,6 +1,7 @@ // SPDX-License-Identifier: Apache-2.0 import { describe, expect, it } from 'vitest'; +import { ActiveRouteAttestationError } from '../src/active-route.js'; import type { BridgeMutationPlan, BridgeSnapshot, @@ -26,20 +27,24 @@ import { } from '../src/provision.js'; import { deploymentSpecDigest } from '../src/spec-digest.js'; import type { + ActiveRouteAttestation, ApplicationR2BucketSnapshot, ApplicationR2Resource, DatabaseReference, DeploymentSecrets, DeploymentSpec, + ExternalMutationFence, ExternalReleaseSnapshot, FleetRecord, FleetStateLease, FleetStateStore, ForceDecommissionStep, + InitialExecutionFenceState, LiveDeployment, MaintenanceHealth, ProvisioningBackend, ProvisioningBackendKind, + SeedDeploymentIdentityOptions, } from '../src/types.js'; import { externalReleaseScriptName } from '../src/workers-for-platforms-backend.js'; @@ -182,6 +187,9 @@ class CommitThenThrowStore extends MemoryStore { } } +/** Pinned so an attestation these fakes return is comparable by value. */ +const ATTESTED_AT = '2026-08-11T00:00:00.000Z'; + const maintenance: MaintenanceHealth = { armed: true, nextAlarmAt: 2_000, @@ -196,11 +204,15 @@ class FakeBackend implements ProvisioningBackend { failAt: string | undefined; cleanupFailAt: string | undefined; live: LiveDeployment | undefined; + /** Strands the route on something else; unset attests the live deployment. */ + activeRoute: ActiveRouteAttestation | undefined; exportLocation = 'r2://fleet-exports/acme.sql'; databaseExists = false; databaseId = 'database-id'; databaseName = 'acme-production'; databaseOwner: string | undefined; + /** Every fence state provisioning asked for, in call order. */ + readonly seededFenceStates: InitialExecutionFenceState[] = []; readonly databaseIdsRead: string[] = []; findDatabaseCalls = 0; retainedReleases: readonly ExternalReleaseSnapshot[] = []; @@ -274,10 +286,17 @@ class FakeBackend implements ProvisioningBackend { return { id: 'database-id', name: 'acme-production', created: true }; } + // The fence state is DECLARED here, not dropped: a fake that omits the + // options argument still satisfies the interface (TypeScript accepts a + // fewer-argument function in a more-argument slot), so a silently unthreaded + // fence state would leave every one of these tests green. async seedDeploymentIdentity( _database: DatabaseReference, tenantTag: string, + _fence: ExternalMutationFence, + options: SeedDeploymentIdentityOptions, ): Promise { + this.seededFenceStates.push(options.initialExecutionFenceState); this.#event('identity'); this.databaseOwner = tenantTag; } @@ -460,6 +479,31 @@ class FakeBackend implements ProvisioningBackend { return completeLiveDeployment(live); } + async attestActiveRoute( + deployment: DeploymentSpec, + ): Promise { + this.#event('attest'); + const live = this.live; + const attestation = + this.activeRoute ?? + (live + ? { + specDigest: live.desiredSpecDigest, + artifactVersion: live.artifactVersion, + physicalScriptName: live.scriptName, + source: 'workers-deployments' as const, + observedAt: ATTESTED_AT, + } + : undefined); + if (!attestation) { + throw new ActiveRouteAttestationError( + `nothing serves '${deployment.routeHostname}'`, + {}, + ); + } + return attestation; + } + async removeTraffic( _deployment?: DeploymentSpec, retainedReleases: readonly ExternalReleaseSnapshot[] = [], @@ -722,6 +766,7 @@ describe('fleet provisioning', () => { const backend = new FakeBackend(); const store = new MemoryStore(); const initial = await provisionDeployment({ + initialExecutionFenceState: 'open', backend, store, spec: spec(), @@ -737,7 +782,13 @@ describe('fleet provisioning', () => { backend.events.length = 0; await expect( - provisionDeployment({ backend, store, spec: spec(), secrets }), + provisionDeployment({ + initialExecutionFenceState: 'open', + backend, + store, + spec: spec(), + secrets, + }), ).rejects.toThrow(/active backend switch 'domain-detach-authorized'/); expect(backend.events).toEqual([]); }); @@ -746,6 +797,7 @@ describe('fleet provisioning', () => { const backend = new FakeBackend(); const store = new MemoryStore(); const result = await provisionDeployment({ + initialExecutionFenceState: 'open', backend, store, spec: spec(), @@ -765,6 +817,9 @@ describe('fleet provisioning', () => { 'inspect', 'promote', 'inspect', + // The ready commit records what is ROUTED, so the last provider read on + // the create path is the attestation, after the promotion, not before. + 'attest', ]); expect(result.record).toMatchObject({ phase: 'ready', @@ -788,6 +843,119 @@ describe('fleet provisioning', () => { 'publishing', 'ready', ]); + expect(backend.seededFenceStates).toEqual(['open']); + }); + + it('commits the routed artifact version, and refuses when it is foreign', async () => { + // #given a first deploy whose route ends up serving a different artifact + const backend = new FakeBackend(); + const store = new MemoryStore(); + backend.activeRoute = { + specDigest: deploymentSpecDigest(spec()), + artifactVersion: 'artifact-someone-else-promoted', + physicalScriptName: 'acme-production', + source: 'workers-deployments', + observedAt: ATTESTED_AT, + }; + + // #when the deployment is provisioned + const provision = provisionDeployment({ + initialExecutionFenceState: 'open', + backend, + store, + spec: spec(), + secrets, + clock: () => 1_000, + routeAttestation: { convergenceBudgetMs: 1, initialRetryDelayMs: 1 }, + }); + + // #then the ready commit refuses rather than recording a version this + // deployment uploaded as the version it serves, and leaves the deployment + // in the phase a retry resumes from + const failure = await provision.catch((error: unknown) => error); + expect(failure).toBeInstanceOf(ProvisioningError); + expect(String((failure as ProvisioningError).cause)).toMatch( + /did not converge/, + ); + expect(store.record?.phase).toBe('publishing'); + + // #and once the route serves the deployed artifact, that is what commits + backend.activeRoute = undefined; + const settled = await provisionDeployment({ + initialExecutionFenceState: 'open', + backend, + store, + spec: spec(), + secrets, + clock: () => 1_000, + }); + expect(settled.record).toMatchObject({ + phase: 'ready', + artifactVersion: 'artifact-v3', + }); + }); + + it('provisions a migration target already locked, exactly once', async () => { + // The capability the fence exists for: a deployment that will RECEIVE a + // migration comes up unable to execute, so nothing starts on it between + // provisioning and the operator's own reopen. + const backend = new FakeBackend(); + const store = new MemoryStore(); + const result = await provisionDeployment({ + initialExecutionFenceState: 'migration-locked', + backend, + store, + spec: spec(), + secrets, + clock: () => 1_000, + }); + + expect(result.record.phase).toBe('ready'); + expect(backend.seededFenceStates).toEqual(['migration-locked']); + }); + + it('rejects an illegal birth fence state before creating ANY resource', async () => { + // #given — the protocol validates this too, but only when the seeding + // statements are built, which is after `database-created`. Refusing at the + // entry is what keeps a typo from costing a Worker and a D1 database first + // and leaving a half-provisioned deployment behind. 'draining' and + // 'proof-only' are real fence states but not coherent BIRTH states: they + // are transitions out of something that already exists. + for (const illegal of ['draining', 'proof-only', 'open ', '', undefined]) { + const backend = new FakeBackend(); + const store = new MemoryStore(); + await expect( + provisionDeployment({ + initialExecutionFenceState: + illegal as unknown as InitialExecutionFenceState, + backend, + store, + spec: spec(), + secrets, + clock: () => 1_000, + }), + ).rejects.toThrow(/initialExecutionFenceState must be one of/); + // #then — no lease taken, no provider call made, nothing persisted. + expect(store.leaseCalls).toBe(0); + expect(backend.events).toEqual([]); + expect(store.record).toBeUndefined(); + } + }); + + it('carries the requested fence state on a failed provisioning pass', async () => { + const backend = new FakeBackend(); + backend.failAt = 'migrations'; + const store = new MemoryStore(); + await expect( + provisionDeployment({ + initialExecutionFenceState: 'migration-locked', + backend, + store, + spec: spec(), + secrets, + }), + ).rejects.toBeInstanceOf(ProvisioningError); + expect(backend.seededFenceStates).toEqual(['migration-locked']); }); it('rejects incomplete or noncontiguous D1 migration history before creating resources', async () => { @@ -810,6 +978,7 @@ describe('fleet provisioning', () => { const backend = new FakeBackend(); await expect( provisionDeployment({ + initialExecutionFenceState: 'open', backend, store: new MemoryStore(), spec: invalid, @@ -827,7 +996,13 @@ describe('fleet provisioning', () => { const deployment = spec(); await expect( - provisionDeployment({ backend, store, spec: deployment, secrets }), + provisionDeployment({ + initialExecutionFenceState: 'open', + backend, + store, + spec: deployment, + secrets, + }), ).rejects.toBeInstanceOf(ProvisioningError); expect(store.record).toMatchObject({ phase: 'database-create-authorized', @@ -838,6 +1013,7 @@ describe('fleet provisioning', () => { await expect( provisionDeployment({ + initialExecutionFenceState: 'open', backend, store, spec: { @@ -852,7 +1028,13 @@ describe('fleet provisioning', () => { backend.failAt = undefined; await expect( - provisionDeployment({ backend, store, spec: deployment, secrets }), + provisionDeployment({ + initialExecutionFenceState: 'open', + backend, + store, + spec: deployment, + secrets, + }), ).resolves.toMatchObject({ record: { phase: 'ready' } }); }); @@ -871,7 +1053,13 @@ describe('fleet provisioning', () => { const store = new ConflictingDatabaseNameStore(); await expect( - provisionDeployment({ backend, store, spec: spec(), secrets }), + provisionDeployment({ + initialExecutionFenceState: 'open', + backend, + store, + spec: spec(), + secrets, + }), ).rejects.toThrow(/UNIQUE constraint failed.*database_name/); expect(backend.events).toEqual([]); expect(store.record).toBeUndefined(); @@ -886,7 +1074,13 @@ describe('fleet provisioning', () => { throw new Error('D1 lookup unavailable before create authorization'); }; await expect( - provisionDeployment({ backend, store, spec: deployment, secrets }), + provisionDeployment({ + initialExecutionFenceState: 'open', + backend, + store, + spec: deployment, + secrets, + }), ).rejects.toBeInstanceOf(ProvisioningError); expect(store.record?.phase).toBe('database-reserved'); @@ -905,7 +1099,13 @@ describe('fleet provisioning', () => { const store = new MemoryStore(); const deployment = spec(); await expect( - provisionDeployment({ backend, store, spec: deployment, secrets }), + provisionDeployment({ + initialExecutionFenceState: 'open', + backend, + store, + spec: deployment, + secrets, + }), ).rejects.toBeInstanceOf(ProvisioningError); backend.failAt = undefined; @@ -916,6 +1116,10 @@ describe('fleet provisioning', () => { cleanupDeploymentArtifacts({ backend, store, spec: deployment }), ).resolves.toBeUndefined(); expect(backend.events).toEqual(['identity', 'delete-database']); + // The freshness proof seeds the most restrictive fence it can: this + // database is about to be deleted, and one that survives a failed delete + // must never come back executing. + expect(backend.seededFenceStates).toEqual(['migration-locked']); expect(backend.databaseExists).toBe(false); expect(store.record).toBeUndefined(); }); @@ -927,7 +1131,13 @@ describe('fleet provisioning', () => { const store = new MemoryStore(); const deployment = spec(); await expect( - provisionDeployment({ backend, store, spec: deployment, secrets }), + provisionDeployment({ + initialExecutionFenceState: 'open', + backend, + store, + spec: deployment, + secrets, + }), ).rejects.toBeInstanceOf(ProvisioningError); expect(store.record?.phase).toBe('database-reserved'); backend.events.length = 0; @@ -944,7 +1154,13 @@ describe('fleet provisioning', () => { const store = new MemoryStore(); const deployment = spec(); await expect( - provisionDeployment({ backend, store, spec: deployment, secrets }), + provisionDeployment({ + initialExecutionFenceState: 'open', + backend, + store, + spec: deployment, + secrets, + }), ).rejects.toBeInstanceOf(ProvisioningError); expect(store.record?.phase).toBe('database-create-authorized'); @@ -952,6 +1168,7 @@ describe('fleet provisioning', () => { backend.databaseOwner = 'other-tenant'; backend.events.length = 0; const failure = await provisionDeployment({ + initialExecutionFenceState: 'open', backend, store, spec: deployment, @@ -977,7 +1194,13 @@ describe('fleet provisioning', () => { const store = new MemoryStore(); await expect( - provisionDeployment({ backend, store, spec: spec(), secrets }), + provisionDeployment({ + initialExecutionFenceState: 'open', + backend, + store, + spec: spec(), + secrets, + }), ).rejects.toBeInstanceOf(ProvisioningError); if (failure === 'identity') { @@ -1000,7 +1223,13 @@ describe('fleet provisioning', () => { const deployment = spec(); await expect( - provisionDeployment({ backend, store, spec: deployment, secrets }), + provisionDeployment({ + initialExecutionFenceState: 'open', + backend, + store, + spec: deployment, + secrets, + }), ).rejects.toBeInstanceOf(ProvisioningError); expect(backend.activeRelease).toEqual({ @@ -1028,7 +1257,13 @@ describe('fleet provisioning', () => { }); await expect( - provisionDeployment({ backend, store, spec: deployment, secrets }), + provisionDeployment({ + initialExecutionFenceState: 'open', + backend, + store, + spec: deployment, + secrets, + }), ).rejects.toBeInstanceOf(ProvisioningError); expect( @@ -1069,6 +1304,7 @@ describe('fleet provisioning', () => { }); const failure = await provisionDeployment({ + initialExecutionFenceState: 'open', backend, store, spec: deployment, @@ -1132,6 +1368,7 @@ describe('fleet provisioning', () => { }); const failure = await provisionDeployment({ + initialExecutionFenceState: 'open', backend, store, spec: deployment, @@ -1192,7 +1429,13 @@ describe('fleet provisioning', () => { }); await expect( - provisionDeployment({ backend, store, spec: deployment, secrets }), + provisionDeployment({ + initialExecutionFenceState: 'open', + backend, + store, + spec: deployment, + secrets, + }), ).rejects.toBeInstanceOf(ProvisioningError); expect( @@ -1219,7 +1462,13 @@ describe('fleet provisioning', () => { }); await expect( - provisionDeployment({ backend, store, spec: external, secrets }), + provisionDeployment({ + initialExecutionFenceState: 'open', + backend, + store, + spec: external, + secrets, + }), ).rejects.toBeInstanceOf(ProvisioningError); expect(store.phases).toContain('platform-resources-deployed'); @@ -1249,7 +1498,13 @@ describe('fleet provisioning', () => { }); await expect( - provisionDeployment({ backend, store, spec: external, secrets }), + provisionDeployment({ + initialExecutionFenceState: 'open', + backend, + store, + spec: external, + secrets, + }), ).rejects.toBeInstanceOf(ProvisioningError); expect(backend.events).toEqual( @@ -1270,6 +1525,7 @@ describe('fleet provisioning', () => { const backend = new FakeBackend('plain-worker'); await expect( provisionDeployment({ + initialExecutionFenceState: 'open', backend, store: new MemoryStore(), spec: spec({ authoredBy: 'external' }), @@ -1283,6 +1539,7 @@ describe('fleet provisioning', () => { const backend = new FakeBackend('plain-worker'); await expect( provisionDeployment({ + initialExecutionFenceState: 'open', backend, store: new MemoryStore(), spec: spec({ @@ -1309,6 +1566,7 @@ describe('fleet provisioning', () => { }); await expect( provisionDeployment({ + initialExecutionFenceState: 'open', backend: new FakeBackend(), store: new MemoryStore(), spec: external, @@ -1339,6 +1597,7 @@ describe('fleet provisioning', () => { const backend = new FakeBackend(); await expect( provisionDeployment({ + initialExecutionFenceState: 'open', backend, store: new MemoryStore(), spec: invalid, @@ -1361,6 +1620,7 @@ describe('fleet provisioning', () => { ], }); const first = await provisionDeployment({ + initialExecutionFenceState: 'open', backend, store, spec: external, @@ -1374,7 +1634,13 @@ describe('fleet provisioning', () => { backend.events.length = 0; await expect( - provisionDeployment({ backend, store, spec: external, secrets }), + provisionDeployment({ + initialExecutionFenceState: 'open', + backend, + store, + spec: external, + secrets, + }), ).rejects.toThrow(/do not match the persisted platform target/); expect(backend.events).toEqual([]); }); @@ -1387,7 +1653,13 @@ describe('fleet provisioning', () => { egressProxyService: undefined, durableObjectMigrations: [], }); - await provisionDeployment({ backend, store, spec: external, secrets }); + await provisionDeployment({ + initialExecutionFenceState: 'open', + backend, + store, + spec: external, + secrets, + }); const current = store.record; if ( !current?.platformResources || @@ -1475,6 +1747,7 @@ describe('fleet provisioning', () => { await expect( provisionDeployment({ + initialExecutionFenceState: 'open', backend, store, spec: external, @@ -1583,6 +1856,7 @@ describe('fleet provisioning', () => { backend.databaseOwner = activeSpec.tenantTag; const result = await provisionDeployment({ + initialExecutionFenceState: 'open', backend, store, spec: activeSpec, @@ -1596,7 +1870,13 @@ describe('fleet provisioning', () => { it('resumes decommissioning after export failure without repeating prior steps', async () => { const backend = new FakeBackend(); const store = new MemoryStore(); - await provisionDeployment({ backend, store, spec: spec(), secrets }); + await provisionDeployment({ + initialExecutionFenceState: 'open', + backend, + store, + spec: spec(), + secrets, + }); backend.events.length = 0; backend.failAt = 'export'; @@ -1636,7 +1916,13 @@ describe('fleet provisioning', () => { r2Buckets: [{ name: 'FILES' }], }, }); - await provisionDeployment({ backend, store, spec: deployment, secrets }); + await provisionDeployment({ + initialExecutionFenceState: 'open', + backend, + store, + spec: deployment, + secrets, + }); backend.events.length = 0; backend.nonempty = true; @@ -1666,7 +1952,13 @@ describe('fleet provisioning', () => { r2Buckets: [{ name: 'FILES' }], }, }); - await provisionDeployment({ backend, store, spec: deployment, secrets }); + await provisionDeployment({ + initialExecutionFenceState: 'open', + backend, + store, + spec: deployment, + secrets, + }); backend.events.length = 0; backend.writeAfterTrafficRemovalOnce = true; @@ -1698,7 +1990,13 @@ describe('fleet provisioning', () => { r2Buckets: [{ name: 'FILES' }], }, }); - await provisionDeployment({ backend, store, spec: deployment, secrets }); + await provisionDeployment({ + initialExecutionFenceState: 'open', + backend, + store, + spec: deployment, + secrets, + }); backend.events.length = 0; backend.failRemoveTrafficResponseOnce = true; @@ -1726,7 +2024,13 @@ describe('fleet provisioning', () => { r2Buckets: [{ name: 'FILES' }], }, }); - await provisionDeployment({ backend, store, spec: deployment, secrets }); + await provisionDeployment({ + initialExecutionFenceState: 'open', + backend, + store, + spec: deployment, + secrets, + }); backend.events.length = 0; store.failAfterCommittedPhase = 'traffic-removed'; @@ -1756,7 +2060,13 @@ describe('fleet provisioning', () => { backend.failAt = 'promote'; await expect( - provisionDeployment({ backend, store, spec: deployment, secrets }), + provisionDeployment({ + initialExecutionFenceState: 'open', + backend, + store, + spec: deployment, + secrets, + }), ).rejects.toThrow(/publishing state is preserved/u); expect(store.record?.phase).toBe('publishing'); expect(backend.live).toBeDefined(); @@ -1782,7 +2092,13 @@ describe('fleet provisioning', () => { const backend = new FakeBackend('plain-worker'); const store = new MemoryStore(); const deployment = spec(); - await provisionDeployment({ backend, store, spec: deployment, secrets }); + await provisionDeployment({ + initialExecutionFenceState: 'open', + backend, + store, + spec: deployment, + secrets, + }); if (!store.record) throw new Error('missing provisioned record'); store.record = { ...store.record, phase: 'worker-deployed' }; backend.events.length = 0; @@ -1806,7 +2122,13 @@ describe('fleet provisioning', () => { it('never deletes the database when export fails', async () => { const backend = new FakeBackend(); const store = new MemoryStore(); - await provisionDeployment({ backend, store, spec: spec(), secrets }); + await provisionDeployment({ + initialExecutionFenceState: 'open', + backend, + store, + spec: spec(), + secrets, + }); backend.events.length = 0; backend.failAt = 'export'; await expect( @@ -1822,7 +2144,13 @@ describe('fleet provisioning', () => { const backend = new FakeBackend(); const store = new MemoryStore(); const deployment = spec(); - await provisionDeployment({ backend, store, spec: deployment, secrets }); + await provisionDeployment({ + initialExecutionFenceState: 'open', + backend, + store, + spec: deployment, + secrets, + }); if (!store.record) throw new Error('missing provisioned record'); const activeRelease: ExternalReleaseSnapshot = { physicalScriptName: 'acme-active-release', @@ -1871,6 +2199,7 @@ describe('fleet provisioning', () => { durableObjectMigrations: [], }); await provisionDeployment({ + initialExecutionFenceState: 'open', backend, store, spec: initialSpec, @@ -1942,7 +2271,13 @@ describe('fleet provisioning', () => { it('recovers when final state persistence fails after D1 deletion', async () => { const backend = new FakeBackend(); const store = new MemoryStore(); - await provisionDeployment({ backend, store, spec: spec(), secrets }); + await provisionDeployment({ + initialExecutionFenceState: 'open', + backend, + store, + spec: spec(), + secrets, + }); store.failPutPhase = 'decommissioned'; await expect( @@ -1969,7 +2304,13 @@ describe('fleet provisioning', () => { it('rejects a persisted database ID that now has a different name', async () => { const backend = new FakeBackend(); const store = new MemoryStore(); - await provisionDeployment({ backend, store, spec: spec(), secrets }); + await provisionDeployment({ + initialExecutionFenceState: 'open', + backend, + store, + spec: spec(), + secrets, + }); backend.databaseName = 'unexpected-database-name'; backend.events.length = 0; @@ -1983,7 +2324,13 @@ describe('fleet provisioning', () => { it('rejects database cleanup when the persisted ID has another sentinel owner', async () => { const backend = new FakeBackend(); const store = new MemoryStore(); - await provisionDeployment({ backend, store, spec: spec(), secrets }); + await provisionDeployment({ + initialExecutionFenceState: 'open', + backend, + store, + spec: spec(), + secrets, + }); if (!store.record) throw new Error('missing test record'); store.record = { ...store.record, phase: 'worker-deployed' }; backend.databaseOwner = 'other-tenant'; @@ -1999,7 +2346,13 @@ describe('fleet provisioning', () => { it('converges cleanup when the persisted database ID is positively absent', async () => { const backend = new FakeBackend(); const store = new MemoryStore(); - await provisionDeployment({ backend, store, spec: spec(), secrets }); + await provisionDeployment({ + initialExecutionFenceState: 'open', + backend, + store, + spec: spec(), + secrets, + }); if (!store.record) throw new Error('missing test record'); store.record = { ...store.record, phase: 'worker-deployed' }; backend.databaseExists = false; @@ -2015,7 +2368,13 @@ describe('fleet provisioning', () => { it('does not treat a persisted-ID lookup failure as database absence', async () => { const backend = new FakeBackend(); const store = new MemoryStore(); - await provisionDeployment({ backend, store, spec: spec(), secrets }); + await provisionDeployment({ + initialExecutionFenceState: 'open', + backend, + store, + spec: spec(), + secrets, + }); if (!store.record) throw new Error('missing test record'); store.record = { ...store.record, phase: 'worker-deployed' }; backend.getDatabase = async () => { @@ -2037,6 +2396,7 @@ describe('fleet provisioning', () => { const store = new MemoryStore(); const failure = await provisionDeployment({ + initialExecutionFenceState: 'open', backend, store, spec: spec(), @@ -2055,7 +2415,13 @@ describe('fleet provisioning', () => { const store = new MemoryStore(); await expect( - provisionDeployment({ backend, store, spec: spec(), secrets }), + provisionDeployment({ + initialExecutionFenceState: 'open', + backend, + store, + spec: spec(), + secrets, + }), ).rejects.toBeInstanceOf(ProvisioningError); expect(store.record?.phase).toBe('database-created'); @@ -2064,7 +2430,13 @@ describe('fleet provisioning', () => { backend.events.length = 0; backend.databaseIdsRead.length = 0; await expect( - provisionDeployment({ backend, store, spec: spec(), secrets }), + provisionDeployment({ + initialExecutionFenceState: 'open', + backend, + store, + spec: spec(), + secrets, + }), ).resolves.toMatchObject({ record: { phase: 'ready' } }); expect(backend.events).not.toContain('database'); expect(backend.events).toContain('identity'); @@ -2074,7 +2446,13 @@ describe('fleet provisioning', () => { it('does not delete a database after an export without integrity evidence', async () => { const backend = new FakeBackend(); const store = new MemoryStore(); - await provisionDeployment({ backend, store, spec: spec(), secrets }); + await provisionDeployment({ + initialExecutionFenceState: 'open', + backend, + store, + spec: spec(), + secrets, + }); backend.events.length = 0; backend.exportDatabase = async () => ({ databaseId: 'database-id', @@ -2092,7 +2470,13 @@ describe('fleet provisioning', () => { it('rejects an export for a different database before persisting or deleting', async () => { const backend = new FakeBackend(); const store = new MemoryStore(); - await provisionDeployment({ backend, store, spec: spec(), secrets }); + await provisionDeployment({ + initialExecutionFenceState: 'open', + backend, + store, + spec: spec(), + secrets, + }); backend.events.length = 0; backend.exportDatabase = async () => ({ databaseId: 'replacement-database-id', @@ -2111,7 +2495,13 @@ describe('fleet provisioning', () => { it('force-decommissions a deployment without a specification and removes its ledger row', async () => { const backend = new FakeBackend('plain-worker'); const store = new MemoryStore(); - await provisionDeployment({ backend, store, spec: spec(), secrets }); + await provisionDeployment({ + initialExecutionFenceState: 'open', + backend, + store, + spec: spec(), + secrets, + }); backend.events.length = 0; const auditEvents: unknown[] = []; @@ -2154,7 +2544,13 @@ describe('fleet provisioning', () => { throw new Error('D1 lookup unavailable before create authorization'); }; await expect( - provisionDeployment({ backend, store, spec: spec(), secrets }), + provisionDeployment({ + initialExecutionFenceState: 'open', + backend, + store, + spec: spec(), + secrets, + }), ).rejects.toBeInstanceOf(ProvisioningError); expect(store.record?.phase).toBe('database-reserved'); backend.events.length = 0; @@ -2177,7 +2573,13 @@ describe('fleet provisioning', () => { const store = new MemoryStore(); backend.failAt = 'database'; await expect( - provisionDeployment({ backend, store, spec: spec(), secrets }), + provisionDeployment({ + initialExecutionFenceState: 'open', + backend, + store, + spec: spec(), + secrets, + }), ).rejects.toBeInstanceOf(ProvisioningError); expect(store.record).toMatchObject({ phase: 'database-create-authorized', @@ -2221,6 +2623,7 @@ describe('fleet provisioning', () => { const terminalStore = new MemoryStore(); await provisionDeployment({ + initialExecutionFenceState: 'open', backend, store: terminalStore, spec: spec(), @@ -2263,6 +2666,7 @@ describe('fleet provisioning', () => { const normalBackend = new FakeBackend(); const normalStore = new MemoryStore(); await provisionDeployment({ + initialExecutionFenceState: 'open', backend: normalBackend, store: normalStore, spec: spec(), @@ -2271,6 +2675,7 @@ describe('fleet provisioning', () => { const forceBackend = new FakeBackend('plain-worker'); const forceStore = new MemoryStore(); await provisionDeployment({ + initialExecutionFenceState: 'open', backend: forceBackend, store: forceStore, spec: spec(), @@ -2308,7 +2713,13 @@ describe('fleet provisioning', () => { it('re-enters a force decommission wedged after traffic removal', async () => { const backend = new FakeBackend('plain-worker'); const store = new MemoryStore(); - await provisionDeployment({ backend, store, spec: spec(), secrets }); + await provisionDeployment({ + initialExecutionFenceState: 'open', + backend, + store, + spec: spec(), + secrets, + }); backend.forceFailOnceAt = 'revoke-credentials'; await expect( @@ -2342,7 +2753,13 @@ describe('fleet provisioning', () => { it('converges after D1 deletion succeeds but the terminal state write fails', async () => { const backend = new FakeBackend('plain-worker'); const store = new MemoryStore(); - await provisionDeployment({ backend, store, spec: spec(), secrets }); + await provisionDeployment({ + initialExecutionFenceState: 'open', + backend, + store, + spec: spec(), + secrets, + }); store.failPutPhase = 'decommissioned'; await expect( @@ -2373,7 +2790,13 @@ describe('fleet provisioning', () => { it('serializes force decommission against concurrent provisioning', async () => { const backend = new FakeBackend('plain-worker'); const store = new MemoryStore(); - await provisionDeployment({ backend, store, spec: spec(), secrets }); + await provisionDeployment({ + initialExecutionFenceState: 'open', + backend, + store, + spec: spec(), + secrets, + }); let releaseStep: (() => void) | undefined; let stepStarted: (() => void) | undefined; const started = new Promise((resolve) => { @@ -2392,7 +2815,13 @@ describe('fleet provisioning', () => { }); await started; await expect( - provisionDeployment({ backend, store, spec: spec(), secrets }), + provisionDeployment({ + initialExecutionFenceState: 'open', + backend, + store, + spec: spec(), + secrets, + }), ).rejects.toThrow(/already being modified/); releaseStep?.(); await expect(force).resolves.toBeUndefined(); @@ -2401,7 +2830,13 @@ describe('fleet provisioning', () => { it('silently removes a completed force teardown after audit delivery fails', async () => { const backend = new FakeBackend('plain-worker'); const store = new MemoryStore(); - await provisionDeployment({ backend, store, spec: spec(), secrets }); + await provisionDeployment({ + initialExecutionFenceState: 'open', + backend, + store, + spec: spec(), + secrets, + }); let auditAttempts = 0; const input = { backend, @@ -2440,6 +2875,7 @@ describe('fleet provisioning', () => { }); const first = provisionDeployment({ + initialExecutionFenceState: 'open', backend, store, spec: spec(), @@ -2447,7 +2883,13 @@ describe('fleet provisioning', () => { }); await Promise.resolve(); await expect( - provisionDeployment({ backend, store, spec: spec(), secrets }), + provisionDeployment({ + initialExecutionFenceState: 'open', + backend, + store, + spec: spec(), + secrets, + }), ).rejects.toThrow(/already being modified/); releaseDatabase?.({ id: 'database-id', @@ -2463,7 +2905,13 @@ describe('fleet provisioning', () => { backend.cleanupFailAt = 'delete-worker'; const store = new MemoryStore(); await expect( - provisionDeployment({ backend, store, spec: spec(), secrets }), + provisionDeployment({ + initialExecutionFenceState: 'open', + backend, + store, + spec: spec(), + secrets, + }), ).rejects.toBeInstanceOf(ProvisioningError); expect(store.record?.phase).toBe('worker-deployed'); @@ -2472,7 +2920,13 @@ describe('fleet provisioning', () => { backend.live = undefined; backend.events.length = 0; await expect( - provisionDeployment({ backend, store, spec: spec(), secrets }), + provisionDeployment({ + initialExecutionFenceState: 'open', + backend, + store, + spec: spec(), + secrets, + }), ).resolves.toMatchObject({ record: { phase: 'ready' } }); expect(backend.events.filter((event) => event === 'worker')).toHaveLength( 1, @@ -2486,7 +2940,13 @@ describe('fleet provisioning', () => { backend.cleanupFailAt = 'delete-worker'; const store = new MemoryStore(); await expect( - provisionDeployment({ backend, store, spec: spec(), secrets }), + provisionDeployment({ + initialExecutionFenceState: 'open', + backend, + store, + spec: spec(), + secrets, + }), ).rejects.toBeInstanceOf(ProvisioningError); backend.failAt = undefined; backend.cleanupFailAt = undefined; @@ -2494,6 +2954,7 @@ describe('fleet provisioning', () => { await expect( provisionDeployment({ + initialExecutionFenceState: 'open', backend, store, spec: spec({ @@ -2510,7 +2971,13 @@ describe('fleet provisioning', () => { it('does not delete D1 when partial cleanup cannot remove the Worker', async () => { const backend = new FakeBackend(); const store = new MemoryStore(); - await provisionDeployment({ backend, store, spec: spec(), secrets }); + await provisionDeployment({ + initialExecutionFenceState: 'open', + backend, + store, + spec: spec(), + secrets, + }); if (!store.record) throw new Error('missing test record'); store.record = { ...store.record, phase: 'worker-deployed' }; backend.cleanupFailAt = 'delete-worker'; diff --git a/packages/fleet-control/test/state-store.test.ts b/packages/fleet-control/test/state-store.test.ts index 4897c4b..332664c 100644 --- a/packages/fleet-control/test/state-store.test.ts +++ b/packages/fleet-control/test/state-store.test.ts @@ -8,6 +8,7 @@ import { externalStateScriptName, } from '../src/platform-resources.js'; import { + ADDED_NULLABLE_TEXT_COLUMNS, D1FleetStateStore, type FleetStateDatabase, } from '../src/state-store.js'; @@ -16,6 +17,14 @@ import type { FleetRecord, PlatformPlaneResourceSet } from '../src/types.js'; const MAINTENANCE_PUBLIC_KEY = '{"kty":"OKP","crv":"Ed25519","alg":"EdDSA","kid":"fleet-maintenance-v1","x":"Lhp1XFeTJJx8FLOCKpn4nkO-tWuZZxXX8ziw0LEvUZo"}'; +function nullableTextColumn(name: string): Readonly> { + return { name, type: 'TEXT', notnull: 0, pk: 0 }; +} + +function addedColumnName(sql: string): string | undefined { + return /ADD COLUMN (?[a-z_]+) TEXT/u.exec(sql)?.groups?.name; +} + class MemoryD1 implements FleetStateDatabase { row: Readonly> | undefined; readonly claims = new Map>>(); @@ -25,14 +34,9 @@ class MemoryD1 implements FleetStateDatabase { bindings: readonly unknown[] = [], ): Promise>[]> { if (sql.startsWith('PRAGMA table_info(anchorage_fleet_deployments)')) { - return [ - { - name: 'backend_switch_intent', - type: 'TEXT', - notnull: 0, - pk: 0, - }, - ]; + return ADDED_NULLABLE_TEXT_COLUMNS.map((name) => + nullableTextColumn(name), + ); } if (sql.startsWith('SELECT *')) return this.row ? [this.row] : []; if (sql.startsWith('INSERT INTO anchorage_fleet_leases')) { @@ -159,6 +163,7 @@ class MemoryD1 implements FleetStateDatabase { 'database_export_location', 'database_export_sha256', 'database_export_size', + 'settled_settlement_key', 'updated_at', ]; this.row = Object.fromEntries( @@ -283,12 +288,9 @@ class MemoryD1 implements FleetStateDatabase { } class SchemaD1 implements FleetStateDatabase { - column: Readonly> | undefined = { - name: 'backend_switch_intent', - type: 'TEXT', - notnull: 0, - pk: 0, - }; + readonly columns = new Map>>( + ADDED_NULLABLE_TEXT_COLUMNS.map((name) => [name, nullableTextColumn(name)]), + ); createAttempts = 0; alterAttempts = 0; failCreateOnce = false; @@ -297,9 +299,7 @@ class SchemaD1 implements FleetStateDatabase { async query( sql: string, ): Promise>[]> { - if (sql.startsWith('PRAGMA table_info')) { - return this.column ? [this.column] : []; - } + if (sql.startsWith('PRAGMA table_info')) return [...this.columns.values()]; return []; } @@ -313,18 +313,14 @@ class SchemaD1 implements FleetStateDatabase { throw new Error('transient schema failure'); } } - if (sql.startsWith('ALTER TABLE')) { + const added = addedColumnName(sql); + if (added) { this.alterAttempts += 1; if (this.failAlterOnce) { this.failAlterOnce = false; throw new Error('genuine migration failure'); } - this.column = { - name: 'backend_switch_intent', - type: 'TEXT', - notnull: 0, - pk: 0, - }; + this.columns.set(added, nullableTextColumn(added)); } } @@ -336,13 +332,15 @@ class SchemaD1 implements FleetStateDatabase { } class ConcurrentSchemaD1 extends SchemaD1 { + /** Every column an ALTER found already present — a real racer's signature. */ + readonly duplicates: string[] = []; readonly #bothInspected: Promise; #releaseInspections!: () => void; #coldInspections = 0; constructor() { super(); - this.column = undefined; + this.columns.clear(); this.#bothInspected = new Promise((resolve) => { this.#releaseInspections = resolve; }); @@ -361,12 +359,12 @@ class ConcurrentSchemaD1 extends SchemaD1 { } override async execute(sql: string): Promise { - if (sql.startsWith('ALTER TABLE') && this.column !== undefined) { + const added = addedColumnName(sql); + if (added && this.columns.has(added)) { this.alterAttempts += 1; + this.duplicates.push(added); throw new Error('D1 migration failed', { - cause: new Error( - 'duplicate column name: backend_switch_intent: SQLITE_ERROR', - ), + cause: new Error(`duplicate column name: ${added}: SQLITE_ERROR`), }); } await super.execute(sql); @@ -432,18 +430,22 @@ describe('D1FleetStateStore release state', () => { second.get('other', 'production'), ]), ).resolves.toEqual([undefined, undefined]); - expect(db.alterAttempts).toBe(2); - expect(db.column).toMatchObject({ - name: 'backend_switch_intent', - type: 'TEXT', - notnull: 0, - pk: 0, - }); + // #then the losing racer's duplicate-column error was swallowed, and every + // upgraded column ended well-formed rather than half-applied + expect(db.duplicates.length).toBeGreaterThan(0); + for (const name of ADDED_NULLABLE_TEXT_COLUMNS) { + expect(db.columns.get(name)).toMatchObject({ + name, + type: 'TEXT', + notnull: 0, + pk: 0, + }); + } }); it('propagates a genuine migration failure and retries it later', async () => { const db = new SchemaD1(); - db.column = undefined; + db.columns.clear(); db.failAlterOnce = true; const store = new D1FleetStateStore(db, { accountId: 'account' }); @@ -451,7 +453,11 @@ describe('D1FleetStateStore release state', () => { /genuine migration failure/u, ); await expect(store.get('acme', 'production')).resolves.toBeUndefined(); - expect(db.alterAttempts).toBe(2); + // The failed column is retried, and every other upgraded column still runs. + expect(db.alterAttempts).toBe(ADDED_NULLABLE_TEXT_COLUMNS.length + 1); + expect([...db.columns.keys()].sort()).toEqual( + [...ADDED_NULLABLE_TEXT_COLUMNS].sort(), + ); }); it('verifies current column shape and does not repeat migration work', async () => { @@ -465,12 +471,12 @@ describe('D1FleetStateStore release state', () => { expect(current.alterAttempts).toBe(0); const incompatible = new SchemaD1(); - incompatible.column = { - name: 'backend_switch_intent', + incompatible.columns.set('settled_settlement_key', { + name: 'settled_settlement_key', type: 'INTEGER', notnull: 1, pk: 0, - }; + }); await expect( new D1FleetStateStore(incompatible, { accountId: 'account' }).get( 'acme', @@ -850,6 +856,55 @@ describe('D1FleetStateStore release state', () => { expect(db.row?.backend_switch_intent).toBeTypeOf('string'); }); + it('round-trips the settled settlement key and refuses a malformed one', async () => { + // #given a deployment that has completed a settlement + const db = new MemoryD1(); + const store = new D1FleetStateStore(db, { accountId: 'account' }); + const settledSettlementKey = '7'.repeat(64); + const record: FleetRecord = { + tenantTag: 'acme', + environment: 'production', + backend: 'plain-worker', + scriptName: 'acme-production', + databaseId: 'db-acme', + databaseName: 'acme-production', + schemaVersion: 1, + artifactVersion: 'artifact-v1', + desiredSpecDigest: 'a'.repeat(64), + durableObjectBindings: [], + routeHostname: 'acme.example.test', + phase: 'ready', + settledSettlementKey, + updatedAt: '2026-08-10T00:00:00.000Z', + }; + + // #when it is written and read back + await store.withDeploymentLease('acme', 'production', (lease) => + lease.put(record), + ); + + // #then the key survives the row, in its own column rather than folded + // into a blob a later reader would have to guess at + await expect(store.get('acme', 'production')).resolves.toMatchObject( + record, + ); + expect(db.row?.settled_settlement_key).toBe(settledSettlementKey); + + // #and a row carrying something that is not a key is refused rather than + // read as "settled", which would silently suppress a real settlement + db.row = { ...db.row, settled_settlement_key: 'not-a-key' }; + await expect(store.get('acme', 'production')).rejects.toThrow( + /invalid settled_settlement_key/, + ); + + // #and an absent key reads as "never settled", so a database written + // before this column existed settles once more rather than never + db.row = { ...db.row, settled_settlement_key: null }; + const unsettled = await store.get('acme', 'production'); + expect(unsettled).toBeDefined(); + expect(unsettled).not.toHaveProperty('settledSettlementKey'); + }); + it('refuses to use a lease capability for a different deployment key', async () => { const store = new D1FleetStateStore(new MemoryD1(), { accountId: 'account', diff --git a/packages/fleet-control/test/workers-for-platforms-backend.test.ts b/packages/fleet-control/test/workers-for-platforms-backend.test.ts index bc89d5b..9fc05fa 100644 --- a/packages/fleet-control/test/workers-for-platforms-backend.test.ts +++ b/packages/fleet-control/test/workers-for-platforms-backend.test.ts @@ -11,6 +11,7 @@ import { verifyAsymmetricMaintenanceCapability, } from '@proofoftech/flowsafe/host-kit'; import { describe, expect, it, vi } from 'vitest'; +import { ActiveRouteAttestationError } from '../src/active-route.js'; import { WorkerDeploymentError } from '../src/deployment-error.js'; import { canonicalDeploymentEgressPolicy, @@ -901,6 +902,32 @@ class FakeApi implements WorkersForPlatformsApi { } } +/** Pinned so an attestation is comparable by value, clock included. */ +const ATTESTED_AT = '2026-08-11T00:00:00.000Z'; + +function hostRoutingTarget(scriptName: string) { + return { + scriptName, + tenantTag: deployment.tenantTag, + environment: deployment.environment, + ...canonicalDeploymentEgressPolicy({ + policyId: externalPlatformResourceGroupId(deployment), + tenantTag: deployment.tenantTag, + environment: deployment.environment, + allowedHosts: ['api.example.com'], + }), + }; +} + +function attestingBackend(client: FakeApi): WorkersForPlatformsBackend { + return new WorkersForPlatformsBackend({ + namespacedState: NAMESPACED_STATE, + client, + hostRoutingKvId: 'host-routing', + clock: () => Date.parse(ATTESTED_AT), + }); +} + function healthResponse(): Response { return Response.json({ nextSweepAt: 2_000, @@ -1213,6 +1240,7 @@ describe('WorkersForPlatformsBackend', () => { }), }); const result = await provisionDeployment({ + initialExecutionFenceState: 'open', backend, store: new MemoryFleetStore(), spec: audited, @@ -1245,6 +1273,7 @@ describe('WorkersForPlatformsBackend', () => { }); await expect( provisionDeployment({ + initialExecutionFenceState: 'open', backend, store: new MemoryFleetStore(result.record), spec: audited, @@ -1265,6 +1294,7 @@ describe('WorkersForPlatformsBackend', () => { }); await expect( provisionDeployment({ + initialExecutionFenceState: 'open', backend, store: new MemoryFleetStore(result.record), spec: audited, @@ -3107,6 +3137,99 @@ describe('WorkersForPlatformsBackend', () => { backend.assertDatabaseDetached(deployment, record, database, fence), ).resolves.toBeUndefined(); }); + + it('attests the release the host route dispatches to, not the expected one', async () => { + // #given a route left pointing at a release the current spec would not + // produce, which is exactly what a landed rollback or a promotion that + // never finished looks like from outside + const api = new FakeApi(); + const routedScriptName = 'acme-production-superseded'; + const routedSpecDigest = 'e'.repeat(64); + api.routeOwner = hostRoutingTarget(routedScriptName); + api.releaseDigests.set(routedScriptName, routedSpecDigest); + api.releaseArtifactVersions.set(routedScriptName, 'etag-superseded'); + const backend = attestingBackend(api); + + // #when the attestation is read + const attestation = await backend.attestActiveRoute(deployment); + + // #then it reports what is routed, and never the release the spec expects + expect(attestation).toEqual({ + specDigest: routedSpecDigest, + artifactVersion: 'etag-superseded', + physicalScriptName: routedScriptName, + source: 'dispatch-route', + observedAt: ATTESTED_AT, + }); + expect(attestation.physicalScriptName).not.toBe( + externalReleaseScriptName(deployment), + ); + expect(api.inspectedScriptNames).toEqual([routedScriptName]); + }); + + it('refuses to attest a hostname that dispatches nowhere', async () => { + // #given no host routing entry at all + const api = new FakeApi(); + api.routeOwner = undefined; + const backend = attestingBackend(api); + + // #when the attestation is read + const failure = await backend + .attestActiveRoute(deployment) + .catch((error: unknown) => error); + + // #then it refuses, and never falls back to the expected release + expect(failure).toBeInstanceOf(ActiveRouteAttestationError); + expect((failure as ActiveRouteAttestationError).message).toContain( + 'dispatches to no release', + ); + expect((failure as ActiveRouteAttestationError).observed).toEqual({}); + expect(api.inspectedScriptNames).toEqual([]); + }); + + it('refuses to attest a route pointing at an absent release', async () => { + // #given a route naming a release the provider no longer holds + const api = new FakeApi(); + api.routeOwner = hostRoutingTarget('acme-production-deleted'); + api.exists = false; + const backend = attestingBackend(api); + + // #when the attestation is read + const failure = await backend + .attestActiveRoute(deployment) + .catch((error: unknown) => error); + + // #then the refusal names the release the route claimed + expect(failure).toBeInstanceOf(ActiveRouteAttestationError); + expect((failure as ActiveRouteAttestationError).observed).toEqual({ + routedScriptName: 'acme-production-deleted', + }); + }); + + it('refuses to attest a routed release carrying no specification digest', async () => { + // #given a routed release whose fleet digest binding is missing + const api = new FakeApi(); + const routedScriptName = 'acme-production-undigested'; + api.routeOwner = hostRoutingTarget(routedScriptName); + api.releaseDigests.set(routedScriptName, ''); + api.releaseArtifactVersions.set(routedScriptName, 'etag-undigested'); + const backend = attestingBackend(api); + + // #when the attestation is read + const failure = await backend + .attestActiveRoute(deployment) + .catch((error: unknown) => error); + + // #then it refuses rather than attesting an unidentifiable artifact + expect(failure).toBeInstanceOf(ActiveRouteAttestationError); + expect((failure as ActiveRouteAttestationError).message).toContain( + 'no fleet specification digest', + ); + expect((failure as ActiveRouteAttestationError).observed).toEqual({ + routedScriptName, + artifactVersion: 'etag-undigested', + }); + }); }); function providerBindingIdentitiesForTest(inspection: { databaseIds: readonly string[]; diff --git a/packages/fleet-control/test/wrangler-loop-backend.test.ts b/packages/fleet-control/test/wrangler-loop-backend.test.ts index cae01c6..48fca30 100644 --- a/packages/fleet-control/test/wrangler-loop-backend.test.ts +++ b/packages/fleet-control/test/wrangler-loop-backend.test.ts @@ -6,6 +6,7 @@ import { tmpdir } from 'node:os'; import { join } from 'node:path'; import { pathToFileURL } from 'node:url'; import { describe, expect, it, vi } from 'vitest'; +import { ActiveRouteAttestationError } from '../src/active-route.js'; import type { DurableDatabaseExportStore } from '../src/cloudflare-client.js'; import { WorkerDeploymentError } from '../src/deployment-error.js'; import { deploymentSpecDigest } from '../src/spec-digest.js'; @@ -16,10 +17,10 @@ import type { ExternalMutationFence, ExternalReleaseSnapshot, FleetRecord, + PlainWorkerCustomDomain, + PlainWorkerRouteApi, } from '../src/types.js'; import { - type PlainWorkerCustomDomain, - type PlainWorkerRouteApi, plainWorkerIngressModule, WranglerLoopBackend, } from '../src/wrangler-loop-backend.js'; @@ -84,6 +85,9 @@ const activeRelease: ExternalReleaseSnapshot = { releaseSchemaVersion: deployment.schemaVersion, }; +/** Pinned so an attestation is comparable by value, clock included. */ +const ATTESTED_AT = '2026-08-11T00:00:00.000Z'; + const mutationFence: ExternalMutationFence = { mutationLeaseTtlMs: 15 * 60_000, assertOwned: vi.fn(async () => {}), @@ -176,6 +180,18 @@ class FakeRouteApi implements PlainWorkerRouteApi { readonly pattern: string; }> = []; beforeListCustomDomains: (() => void) | undefined; + /** Every script name attestation asked about, in call order. */ + readonly activeRouteReads: string[] = []; + /** The reduced active-route read; undefined models an absent script. */ + activeRoute: + | Readonly<{ artifactVersion: string; specDigest: string | undefined }> + | undefined; + /** + * A refusal the provider read itself raises — an ambiguous traffic split. + * The rule producing it lives in the client and is proven there; what this + * models is the backend's obligation not to swallow or reshape it. + */ + activeRouteFailure: Error | undefined; domains: PlainWorkerCustomDomain[]; constructor(domains: readonly PlainWorkerCustomDomain[] = []) { @@ -225,6 +241,18 @@ class FakeRouteApi implements PlainWorkerRouteApi { this.databasePresent = false; } + async inspectActiveWorkerRoute(scriptName: string): Promise< + | Readonly<{ + artifactVersion: string; + specDigest: string | undefined; + }> + | undefined + > { + this.activeRouteReads.push(scriptName); + if (this.activeRouteFailure) throw this.activeRouteFailure; + return this.activeRoute; + } + async listCustomDomains(): Promise { this.beforeListCustomDomains?.(); this.listCalls += 1; @@ -349,6 +377,7 @@ function backend( readonly exportStore?: DurableDatabaseExportStore; readonly routeApi?: PlainWorkerRouteApi; readonly maintenanceRequestTimeoutMs?: number; + readonly clock?: () => number; } = {}, ): WranglerLoopBackend { const exportStore: DurableDatabaseExportStore = options.exportStore ?? { @@ -376,6 +405,7 @@ function backend( exportStore, fetch: options.fetch, maintenanceRequestTimeoutMs: options.maintenanceRequestTimeoutMs, + clock: options.clock, }); } @@ -1132,6 +1162,9 @@ export default { return []; }, async deleteControlSecrets() {}, + async inspectActiveWorkerRoute(): Promise { + throw new Error('unused'); + }, async listCustomDomains() { reads += 1; return reads === 1 @@ -1792,15 +1825,22 @@ export default { provisioned_at TEXT NOT NULL )`; let sentinelExists = false; + let fenceExists = false; let owner: string | undefined; + let fenceBindings: readonly unknown[] | undefined; const runner = new FakeRunner(); const routeApi = new FakeRouteApi(); routeApi.queryHandler = async (sql, bindings) => { let results: readonly Readonly>[] = []; if (sql.includes("sqlite_schema WHERE type = 'table' ORDER BY name")) { - results = sentinelExists - ? [{ name: 'flowsafe_deployment', sql: sentinelDdl }] - : []; + results = [ + ...(sentinelExists + ? [{ name: 'flowsafe_deployment', sql: sentinelDdl }] + : []), + ...(fenceExists + ? [{ name: 'flowsafe_execution_fence', sql: 'CREATE' }] + : []), + ]; } else if ( sql.includes('name = ?') && bindings[0] === 'flowsafe_deployment' @@ -1814,6 +1854,16 @@ export default { ]; } else if (sql.startsWith('SELECT id, tenant_tag')) { results = owner ? [{ id: 1, tenant_tag: owner }] : []; + } else if ( + // Matched on the TARGET table, ahead of the generic arms: the ownership + // insert names the fence table inside its exclusion list. + sql.startsWith('CREATE TABLE IF NOT EXISTS flowsafe_execution_fence') + ) { + fenceExists = true; + } else if ( + sql.startsWith('INSERT OR IGNORE INTO flowsafe_execution_fence') + ) { + fenceBindings = bindings; } else if (sql.startsWith('CREATE TABLE')) { sentinelExists = true; } else if (sql.startsWith('INSERT OR IGNORE')) { @@ -1826,7 +1876,9 @@ export default { await expect( subject.readDeploymentIdentity(database, mutationFence), ).resolves.toBeUndefined(); - await subject.seedDeploymentIdentity(database, 'acme', mutationFence); + await subject.seedDeploymentIdentity(database, 'acme', mutationFence, { + initialExecutionFenceState: 'migration-locked', + }); await expect( subject.readDeploymentIdentity(database, mutationFence), ).resolves.toBe('acme'); @@ -1837,6 +1889,16 @@ export default { (query) => query.databaseId === database.id, ), ).toBe(true); + // The fence rides the same fenced provider-native path, and every binding + // reaches it as a STRING — the plain-Worker adapter rejects anything else + // (restD1Bindings), which is why the seeded timestamp is bound as text and + // left to SQLite's INTEGER affinity. + expect(fenceExists).toBe(true); + expect(fenceBindings).toEqual([ + 'deployment', + 'migration-locked', + expect.stringMatching(/^\d+$/), + ]); }); it('forwards SQLite literals, identifiers, comments, and numbered parameters unchanged', async () => { @@ -2839,6 +2901,9 @@ export default { return []; }, async deleteControlSecrets() {}, + async inspectActiveWorkerRoute(): Promise { + throw new Error('unused'); + }, async listCustomDomains() { return [ { @@ -3056,4 +3121,108 @@ export default { 'wrangler authentication failed', ); }); + + it('attests the routed version through the provider API, not the CLI', async () => { + // #given a Worker whose deployment reduces to one routed version + const routeApi = new FakeRouteApi(); + routeApi.activeRoute = { + artifactVersion: 'version-routed', + specDigest: deploymentSpecDigest(deployment), + }; + const runner = new FakeRunner(async () => { + throw new Error('attestation must not shell out to wrangler'); + }); + const subject = backend(runner, { + routeApi, + clock: () => Date.parse(ATTESTED_AT), + }); + + // #when the route is attested + const attestation = await subject.attestActiveRoute(deployment); + + // #then the attestation names the routed version and cost no CLI call + expect(attestation).toEqual({ + specDigest: deploymentSpecDigest(deployment), + artifactVersion: 'version-routed', + physicalScriptName: deployment.scriptName, + source: 'workers-deployments', + observedAt: ATTESTED_AT, + }); + expect(routeApi.activeRouteReads).toEqual([deployment.scriptName]); + expect(runner.calls).toEqual([]); + }); + + it('refuses to attest a Worker with no deployment serving traffic', async () => { + // #given a script the provider does not hold + const routeApi = new FakeRouteApi(); + routeApi.activeRoute = undefined; + const subject = backend(new FakeRunner(), { routeApi }); + + // #when the route is attested + const failure = await subject + .attestActiveRoute(deployment) + .catch((error: unknown) => error); + + // #then it refuses rather than reporting an absent deployment as routed + expect(failure).toBeInstanceOf(ActiveRouteAttestationError); + expect((failure as ActiveRouteAttestationError).message).toContain( + 'no deployment serving traffic', + ); + expect((failure as ActiveRouteAttestationError).observed).toEqual({}); + }); + + it('refuses to attest a routed version carrying no specification digest', async () => { + // #given a routed version whose fleet digest binding is missing, then one + // whose binding holds something that is not a digest at all + const routeApi = new FakeRouteApi(); + routeApi.activeRoute = { + artifactVersion: 'version-undigested', + specDigest: undefined, + }; + const subject = backend(new FakeRunner(), { routeApi }); + const attest = () => + subject.attestActiveRoute(deployment).catch((error: unknown) => error); + + // #when each is attested + const missing = await attest(); + routeApi.activeRoute = { + artifactVersion: 'version-undigested', + specDigest: 'not-a-digest', + }; + const malformed = await attest(); + + // #then both refuse with the version they could not identify + for (const failure of [missing, malformed]) { + expect(failure).toBeInstanceOf(ActiveRouteAttestationError); + expect((failure as ActiveRouteAttestationError).observed).toEqual({ + routedScriptName: deployment.scriptName, + artifactVersion: 'version-undigested', + }); + } + }); + + it('passes an ambiguous traffic split through as the provider raised it', async () => { + // #given the provider read refusing an unreduced deployment + const routeApi = new FakeRouteApi(); + const split = new ActiveRouteAttestationError( + "ordinary Worker 'acme-production' must have exactly one current version receiving 100% of traffic", + { + routedScriptName: deployment.scriptName, + trafficSplit: [ + { artifactVersion: 'version-live', percentage: 100 }, + { artifactVersion: 'version-candidate', percentage: 0 }, + ], + }, + ); + routeApi.activeRouteFailure = split; + const subject = backend(new FakeRunner(), { routeApi }); + + // #when the route is attested + const failure = await subject + .attestActiveRoute(deployment) + .catch((error: unknown) => error); + + // #then the refusal reaches the caller intact, percentages included + expect(failure).toBe(split); + }); }); diff --git a/packages/flowsafe/CHANGELOG.md b/packages/flowsafe/CHANGELOG.md index e527475..a1197e8 100644 --- a/packages/flowsafe/CHANGELOG.md +++ b/packages/flowsafe/CHANGELOG.md @@ -1,5 +1,30 @@ # @proofoftech/flowsafe +## 0.20.0 + +### Minor Changes + +- 1212ba5: Flowsafe now exposes deployment-wide execution fencing, owner-bound idempotent starts, and a read-only drain inventory for physical deployment migrations. + + This changes the public runtime and host contract: + + - The execution fence has `open`, `draining`, `migration-locked`, and `proof-only` states. Transitions use compare-and-set through `POST /admin/execution-fence`; a stale expected state returns `409` with `FENCE_CAS_CONFLICT`, and invalid state or proof-key values return `400` with `INVALID_EXECUTION_FENCE_REQUEST`. `GET /admin/execution-fence` reads the current state. Both routes require `MAINTENANCE_ADMIN_SECRET`. + - Fence refusals return `503` with `EXECUTION_FENCED`. `open` admits every entry. `draining` refuses new run mints and future-work authoring, but admits work on existing runs, dispatch of queued background tasks, and new background-task enqueues. `migration-locked` refuses every execution entry, including background-task enqueue, dispatch, and stale-task re-drive. `proof-only` admits only the start carrying the nominated proof key and later work addressed to its bound proof run; it also refuses background-task enqueue, dispatch, and stale-task re-drive. Reads, termination, cancellation, timeout, objective clear, and schedule pause or delete remain available. In-flight compute is not preempted. + - The new `flowsafe_execution_fence` table stores one deployment-wide row. The deployment-identity protocol treats it as protocol-owned state during ownership checks. A database created before this release reads an absent table or row as `open`. + - **BREAKING:** `ExecutionFenceWiring` is required on the non-`'none'` `RunRouterStartIdempotency` arm of `RunRouterOptions.startIdempotency`, and on `HostApprovalServiceOptions`, `ScheduleTickOptions`, `ScheduleRouterOptions`, `SignalProviderHostWiring`, `WebhookRouterOptions`, `ApprovalServiceOptions`, `NotificationDispatchTickOptions`, `StorageInitOptions`, `AgentThreadTopologyOptions`, `ObjectiveRouterOptions`, and `BackgroundTaskHostOptions`. + - **BREAKING:** `provisionDeploymentIdentityProtocol()`, `seedDeploymentIdentity()`, and the `flowsafe-provision --initial-fence-state` CLI require an initial state. Provisioning accepts only `open` or `migration-locked`; there is no default. + - **BREAKING:** `BackgroundTaskHost.manager` is removed. Use `enqueue()`, `getTask()`, `listTasks()`, and `stream()`, and type read-only consumers with `BackgroundTaskReads`. + - **BEHAVIOR CHANGE:** the agent-host and host-kit stream routers preserve structured Durable Object refusals. Clients now receive the original `503` or `409` and a `reason` body instead of a bare `500` for structured server-side refusals. + - Refusal and denial message text no longer exposes internal plan identifiers. Reason codes and statuses are unchanged. + - `idempotencyKey` makes `POST /runs`, trusted agent-host starts, and `streamUntilPersisted()` converge on the same server-minted run. Callers still cannot supply `runId` to `POST /runs`; doing so returns `400`. + - Idempotent-start decisions return `IDEMPOTENT_START_OWNER_MISMATCH` (`403`), `IDEMPOTENT_START_TARGET_MISMATCH` (`409`), `IDEMPOTENT_START_PENDING` (`503`), `IDEMPOTENT_START_UNRESOLVABLE` (`409`), or `IDEMPOTENT_START_ALREADY_SETTLED` (`409`). Operational failures return `IDEMPOTENT_START_UNSUPPORTED` (`503`) or `IDEMPOTENT_START_UNREADABLE` (`503`), and malformed input returns `INVALID_START_IDEMPOTENCY_REQUEST` (`400`). `isStartReservationRefusal()` recognizes the five decision refusals, unsupported wiring, and malformed input; unreadable storage propagates separately. + - A replay returns the original run's persisted state. `IDEMPOTENT_START_PENDING` includes `pendingSince`. `IDEMPOTENT_START_UNRESOLVABLE` is a point-in-time answer, so re-probe before acting. A replayed suspended start omits the start response's `approval` and `approvals` fields. The `flowsafe_start_idempotency` row remains valid for the configured reservation-retention horizon. + - **BREAKING:** `RunRouterOptions`, `AgentThreadTopologyOptions`, and `StorageInitOptions` require `startIdempotency` wiring. `START_IDEMPOTENCY_RETENTION_DAYS` controls how long spent keys remain valid and defaults to run retention. + - `GET /admin/inventory` reports work categories `runs`, `approvals-waiting`, `schedule-deferred-dispatches`, `pending-notifications`, `background-tasks`, `resource-owners`, and `start-reservations`, plus standing categories `schedules` and `signal-subscriptions`. Standing categories are never required to empty. + - `INVENTORY_DRAIN_PROOF` requires every work category to be empty across two consecutive full sweeps at least one 60-second alarm cadence apart, while the fence remains `draining`. Inventory readings are point-in-time observations rather than snapshots and can move in either direction while draining admits work. Empty results cannot over-count, and keyset pagination never skips a row that existed before the sweep began. A host that needs a hard guarantee can re-sweep once after transitioning to `migration-locked`: an empty post-lock sweep is conclusive, while a non-empty one means work is still outstanding, either because it entered after the proof or because the lock parked it before it finished. Return to `draining` and repeat the proof. An inventory read taken under `migration-locked` measures what the fence parked rather than what the deployment would otherwise be doing. `INVENTORY_UNENUMERABLE` declares the run-owner recovery journal and persisted idle signals; the latter deliberately survive migration. `FLOWSAFE_TABLES` keeps every Flowsafe-owned table accounted for. + + When upgrading, pass `'none'` only for leaf hosts with no database, select `open` or `migration-locked` during provisioning, replace `host.manager` reads with the host methods, preserve structured `503` and `409` refusal bodies, and re-probe `IDEMPOTENT_START_UNRESOLVABLE` before deciding whether to use a fresh key. + ## 0.19.0 ### Minor Changes diff --git a/packages/flowsafe/README.md b/packages/flowsafe/README.md index a20215c..874b0ce 100644 --- a/packages/flowsafe/README.md +++ b/packages/flowsafe/README.md @@ -291,16 +291,27 @@ npm install --save-dev "wrangler@>=4.118 <5" npx flowsafe-provision \ --database \ --tag \ + --initial-fence-state open \ --remote \ --config wrangler.jsonc ``` Fleet control planes can import the same fail-closed sentinel implementation from `@proofoftech/flowsafe/deployment-identity-protocol` instead of duplicating its schema or race handling. +### Fence and drain a deployment + +One Flowsafe deployment is one tenant, so the execution fence controls the complete deployment. `open` admits all work. `draining` refuses new run mints and future-work authoring while existing runs and deliveries finish. It still accepts new background-task enqueues and dispatches queued tasks because both are drainable work. Background-task enqueue, dispatch, and stale-task re-drive are refused in `migration-locked` and `proof-only`. Signal wakes that would mint a run persist instead. `migration-locked` refuses execution, and `proof-only` admits only the nominated start and its bound run. The fence never preempts compute already in flight. + +Provisioning requires `--initial-fence-state open` or `--initial-fence-state migration-locked`; it never chooses a default. A pre-0.20 database without a fence row reads as `open`. For locked-at-birth provisioning, read `GET /admin/execution-fence` afterward and fail unless it reports `migration-locked`. + +Use `GET /admin/inventory` while the fence remains `draining`. A drain is proven only after every work category is empty across two complete sweeps at least 60 seconds apart. Readings are point-in-time observations rather than snapshots and can move in either direction while draining admits work. Empty results cannot over-count, and keyset pagination never skips a row that existed before the sweep began. If you need a hard guarantee, re-sweep once after transitioning to `migration-locked`: an empty post-lock sweep is conclusive; a non-empty one means work is still outstanding, either because it entered after the proof or because the lock parked it before it finished. Return to `draining` and repeat the proof. An inventory read taken under `migration-locked` measures what the fence parked rather than what the deployment would otherwise be doing. Schedules and signal subscriptions are standing configuration and need not empty. Persisted idle signals are deliberately unenumerable and carry into the replacement deployment. + ### Runtime ids are opaque The host mints opaque, path-safe run and thread ids. `RunnerRuntime.start()` requires a host-owned run id and has no generation fallback. The id scopes the snapshot, Durable Object, approval lookup, stream address, and artifact path, but it carries no customer identity. +Callers that need exactly-once start behavior supply an `idempotencyKey`, never a run ID. The key is available on `POST /runs`, trusted agent-host starts, and `streamUntilPersisted()`. A retry returns the same persisted run. `IDEMPOTENT_START_PENDING` includes `pendingSince`; re-probe the point-in-time `IDEMPOTENT_START_UNRESOLVABLE` result before acting. A key remains valid until its reservation-retention horizon expires. + ### Stores are deployment-wide `D1ApprovalStoreFactory.store()` and `D1SubscriptionStoreFactory.store()` return the store for the bound database. Tables and indexes contain no tenant column. Legacy pooled schemas require a fresh database. @@ -484,6 +495,10 @@ Complete wiring is in the [durable-agents guide](https://github.com/ProofOfTechO The composed `createFlowsafeWorker()` owns the shared route and maintenance-duty pipeline. Hosts inject workflows, identity verification, topology-backed optional routers, budget wrappers, notification transport, an invocation-scoped artifact-store factory, the storage table prefix, schedule tick, and extra purge duties. +Protect `GET` and `POST /admin/execution-fence` plus `GET /admin/inventory` with a distinct `MAINTENANCE_ADMIN_SECRET`. Fence transitions use CAS and return `409` with `FENCE_CAS_CONFLICT` when the expected state is stale. Fenced execution returns `503` with an `EXECUTION_FENCED` reason. The agent-host and stream routers preserve structured `503` and `409` refusals instead of collapsing them to a generic `500`. + +Every leaf option that accepts `ExecutionFenceWiring` requires an explicit store or `'none'`. Run-router, agent-thread-topology, and storage initialization also require explicit start-idempotency wiring. Use `'none'` only when no database exists. `BackgroundTaskHost` no longer exposes its manager; call `enqueue()`, `getTask()`, `listTasks()`, or `stream()` on the host and use `BackgroundTaskReads` for read-only route composition. + `createFlowsafeMaintenanceDurableObject()` runs deadline expiry, approval service-level agreement (SLA) sweep, retention purge, and optional schedule fire as separate alarm invocations. Provider polling and background recovery use their own Durable Object alarms. Read: diff --git a/packages/flowsafe/deployment-identity-protocol.d.mts b/packages/flowsafe/deployment-identity-protocol.d.mts index d4adf35..ac830e2 100644 --- a/packages/flowsafe/deployment-identity-protocol.d.mts +++ b/packages/flowsafe/deployment-identity-protocol.d.mts @@ -29,8 +29,44 @@ export const DEPLOYMENT_SENTINEL_COLUMNS: readonly Readonly<{ pk: number; }>[]; +/** The table the single deployment execution fence row lives in. */ +export const EXECUTION_FENCE_TABLE: 'flowsafe_execution_fence'; +/** The fence row's fixed primary key — one deployment, one database, one row. */ +export const EXECUTION_FENCE_ROW_ID: 'deployment'; +/** Every fence state, ordered from most to least permissive. */ +export const EXECUTION_FENCE_STATES: readonly [ + 'open', + 'draining', + 'migration-locked', + 'proof-only', +]; +/** The states a deployment may be BORN in. */ +export const INITIAL_EXECUTION_FENCE_STATES: readonly [ + 'open', + 'migration-locked', +]; +/** + * The fence table's schema. `do-runner/execution-fence.ts` issues this exact + * string, so the store and the provisioning protocol cannot create differently + * shaped tables. + */ +export const EXECUTION_FENCE_DDL: string; + +/** The fence state a deployment is provisioned into. Required; no default. */ +export type InitialExecutionFenceState = + (typeof INITIAL_EXECUTION_FENCE_STATES)[number]; + export class DeploymentIdentityError extends Error {} +/** + * Validate the fence state a deployment is to be born in, throwing + * DeploymentIdentityError on anything else. + */ +export function assertInitialExecutionFenceState( + state: unknown, + caller: string, +): InitialExecutionFenceState; + export function assertDeploymentIdentitySecret( secret: unknown, caller?: string, @@ -62,8 +98,16 @@ export function readDeploymentIdentityProtocol( export function provisionDeploymentIdentityProtocol( execute: DeploymentIdentityProtocolExecutor, tag: string, - options?: { + options: { caller?: string; provisionedAt?: string; + /** Injectable clock (epoch milliseconds) for the seeded rows. */ + now?: () => number; + /** + * The fence state the deployment is born in. REQUIRED and without a + * default: a migration host that forgot to ask for 'migration-locked' + * would otherwise silently get an executing deployment. + */ + initialExecutionFenceState: InitialExecutionFenceState; }, ): Promise; diff --git a/packages/flowsafe/deployment-identity-protocol.mjs b/packages/flowsafe/deployment-identity-protocol.mjs index 0e9ea96..054d584 100644 --- a/packages/flowsafe/deployment-identity-protocol.mjs +++ b/packages/flowsafe/deployment-identity-protocol.mjs @@ -26,12 +26,82 @@ export const DEPLOYMENT_SENTINEL_COLUMNS = Object.freeze([ }), ]); +// --- Deployment execution fence (F1) --------------------------------------- +// +// The fence table and its single row are created BY THIS PROTOCOL, so every +// database provisioned from 0.20 on is born carrying an EXPLICIT fence state +// instead of leaning on the absent-row-reads-open upgrade rule. +// +// The vocabulary and the DDL live HERE rather than beside the store that reads +// them (src/do-runner/execution-fence.ts, which imports them from this module) +// for one reason: this file is the only place both sides can share. It ships at +// the package root and is loaded by the provisioning CLI, by fleet-control's +// backends, and by the runtime — none of which can import the package's +// TypeScript sources. And the two MUST be one string: the store's own +// `CREATE TABLE IF NOT EXISTS` silently accepts a differently-shaped table this +// protocol created, so a drifted copy would not fail, it would quietly drop the +// CHECK constraints that make the store's compare-and-sets total. + +/** The table the single fence row lives in — flowsafe-owned, outside `mastra_%`. */ +export const EXECUTION_FENCE_TABLE = 'flowsafe_execution_fence'; + +/** + * The fence row's fixed primary key. The fence is a property of the DEPLOYMENT + * and a deployment is one database, so there is exactly one row and its key is + * a constant. + */ +export const EXECUTION_FENCE_ROW_ID = 'deployment'; + +/** Every fence state, ordered from most to least permissive. */ +export const EXECUTION_FENCE_STATES = Object.freeze([ + 'open', + 'draining', + 'migration-locked', + 'proof-only', +]); + +/** + * The states a deployment may be BORN in. `draining` and `proof-only` are + * transitions out of a state that already exists — draining finishes work a + * fresh database has none of, and proof-only nominates a run nothing has yet + * started — so neither is a coherent initial condition. + */ +export const INITIAL_EXECUTION_FENCE_STATES = Object.freeze([ + 'open', + 'migration-locked', +]); + +export const EXECUTION_FENCE_DDL = `CREATE TABLE IF NOT EXISTS ${EXECUTION_FENCE_TABLE} ( + id TEXT PRIMARY KEY CHECK (id = '${EXECUTION_FENCE_ROW_ID}'), + state TEXT NOT NULL CHECK (state IN (${EXECUTION_FENCE_STATES.map((state) => `'${state}'`).join(', ')})), + proof_key TEXT, + proof_run_id TEXT, + updated_at INTEGER NOT NULL + )`; + const SENTINEL_SQL_PATTERN = /^create table (?:if not exists )?flowsafe_deployment\s*\(\s*id integer primary key check\s*\(\s*id\s*=\s*1\s*\)\s*,\s*tenant_tag text not null\s*,\s*provisioned_at text not null\s*\)$/i; const D1_OWNED_INTERNAL_TABLES = Object.freeze(['_cf_KV', '_cf_METADATA']); +/** + * Tables whose presence does NOT make a database "unowned application state": + * D1's own internal tables plus every table this protocol creates itself. + * + * The fence table belongs here because it can legitimately exist BEFORE the + * ownership row does — a previous provisioning attempt that died between the + * fence DDL and the sentinel insert leaves exactly that residue, and the + * runtime store also materializes the table on its first control-plane + * transition. Without the exclusion the next provisioning pass would read its + * own leftovers as somebody else's application data and refuse the database + * forever (`unownedDatabaseError`), and the conditional ownership insert below + * would never fire. + */ +const NON_APPLICATION_TABLES = Object.freeze([ + ...D1_OWNED_INTERNAL_TABLES, + EXECUTION_FENCE_TABLE, +]); const MIN_DEPLOYMENT_CREDENTIAL_LENGTH = 32; const MAX_DEPLOYMENT_CREDENTIAL_LENGTH = 256; -const D1_OWNED_TABLE_EXCLUSIONS = D1_OWNED_INTERNAL_TABLES.map( +const NON_APPLICATION_TABLE_EXCLUSIONS = NON_APPLICATION_TABLES.map( (name) => ` AND name <> '${name}'`, ).join('\n'); @@ -60,6 +130,11 @@ const CREATE_SENTINEL = Object.freeze({ sql: DEPLOYMENT_SENTINEL_DDL, bindings: Object.freeze([]), }); +const CREATE_EXECUTION_FENCE = Object.freeze({ + mode: 'write', + sql: EXECUTION_FENCE_DDL, + bindings: Object.freeze([]), +}); export class DeploymentIdentityError extends Error { constructor(message) { @@ -106,6 +181,28 @@ export function assertValidDeploymentTag(tag, caller) { } } +/** + * Validate the fence state a deployment is to be born in. + * + * Loud on anything else, and with NO default anywhere above it: the failure + * this closes is a migration host forgetting to ask for 'migration-locked' and + * silently getting an executing deployment, which is exactly the condition a + * migration exists to prevent. Making the argument required turns that into an + * obligation the caller cannot skip, while still letting a host that wants an + * open deployment say so. + */ +export function assertInitialExecutionFenceState(state, caller) { + if ( + typeof state !== 'string' || + !INITIAL_EXECUTION_FENCE_STATES.includes(state) + ) { + throw new DeploymentIdentityError( + `${caller}: initialExecutionFenceState must be one of ${INITIAL_EXECUTION_FENCE_STATES.join(', ')} (got '${String(state)}') — it has no default on purpose`, + ); + } + return state; +} + export function normalizeDeploymentSentinelSql(sql) { return sql.replace(/\s+/g, ' ').trim(); } @@ -118,7 +215,7 @@ export function deploymentIdentityApplicationTables(rows) { typeof name === 'string' && name !== DEPLOYMENT_SENTINEL_TABLE && !name.startsWith('sqlite_') && - !D1_OWNED_INTERNAL_TABLES.includes(name), + !NON_APPLICATION_TABLES.includes(name), ) .sort(); } @@ -170,13 +267,48 @@ function conditionalOwnershipInsert(tag, provisionedAt) { SELECT 1 FROM sqlite_schema WHERE type = 'table' AND name <> '${DEPLOYMENT_SENTINEL_TABLE}' -${D1_OWNED_TABLE_EXCLUSIONS} +${NON_APPLICATION_TABLE_EXCLUSIONS} AND name NOT GLOB 'sqlite_*' )`, bindings: [tag, provisionedAt], }; } +function seedExecutionFenceRow(state, seededAt) { + return { + mode: 'write', + sql: `INSERT OR IGNORE INTO ${EXECUTION_FENCE_TABLE} + (id, state, proof_key, proof_run_id, updated_at) + VALUES (?, ?, NULL, NULL, ?)`, + // INSERT OR IGNORE, never an upsert: seeding runs on every provisioning + // pass, and a re-provision of a LIVE deployment must not silently reopen a + // fence an operator closed. + // + // `updated_at` is an INTEGER column bound as TEXT because D1's REST query + // API takes every parameter as a string (fleet-control's + // d1RestParameters rejects anything else). SQLite's INTEGER affinity + // converts a well-formed integer literal on write, so the column still + // holds a number. + bindings: [EXECUTION_FENCE_ROW_ID, state, String(seededAt)], + }; +} + +/** + * Write the deployment's initial fence row, if it has none. + * + * Two statements rather than one request: every executor this protocol is + * driven through — the runtime's `db.prepare()`, the CLI's + * `wrangler d1 execute --command`, and both fleet-control backends' REST + * `/query` with bound parameters — carries exactly ONE statement per call, so + * there is no seam here through which a batch could be sent. The DDL therefore + * runs first and the row second; a crash between them leaves an empty fence + * table, which reads as `open` and is healed by the next invocation. + */ +async function seedExecutionFence(execute, state, seededAt) { + await execute(CREATE_EXECUTION_FENCE); + await execute(seedExecutionFenceRow(state, seededAt)); +} + async function scanTables(execute) { return execute(SCAN_TABLES); } @@ -214,9 +346,24 @@ function differentOwnerError(caller, stored, tag) { export async function provisionDeploymentIdentityProtocol( execute, tag, - { caller = 'seedDeploymentIdentity', provisionedAt } = {}, + { + caller = 'seedDeploymentIdentity', + provisionedAt, + now = Date.now, + initialExecutionFenceState, + } = {}, ) { assertValidDeploymentTag(tag, caller); + // Validated BEFORE the first statement: a caller that omitted the fence state + // must learn so without having stamped ownership onto a database first. + const fenceState = assertInitialExecutionFenceState( + initialExecutionFenceState, + caller, + ); + // One instant for both rows. The sentinel stores it as ISO TEXT and the fence + // as epoch-milliseconds INTEGER because that is what each column already is; + // an explicit `provisionedAt` still wins for the sentinel, as before. + const seededAt = now(); const tables = await scanTables(execute); const applicationTables = deploymentIdentityApplicationTables(tables); const sentinelExists = tables.some( @@ -227,6 +374,13 @@ export async function provisionDeploymentIdentityProtocol( if (storedBeforeCreate !== tag) { throw differentOwnerError(caller, storedBeforeCreate, tag); } + // The already-owned early return still seeds the fence. A previous pass + // that died between the ownership insert and the fence row left a + // deployment with an owner and NO explicit fence — permanently implicit-open + // residue, on the one deployment a migration most needs to be able to lock. + // Seeding here is what heals it, and INSERT-if-absent is what makes + // repeating it safe on a deployment whose fence has since been moved. + await seedExecutionFence(execute, fenceState, seededAt); return; } if (applicationTables.length > 0) { @@ -239,11 +393,15 @@ export async function provisionDeploymentIdentityProtocol( if (storedAfterCreate !== tag) { throw differentOwnerError(caller, storedAfterCreate, tag); } + await seedExecutionFence(execute, fenceState, seededAt); return; } await execute( - conditionalOwnershipInsert(tag, provisionedAt ?? new Date().toISOString()), + conditionalOwnershipInsert( + tag, + provisionedAt ?? new Date(seededAt).toISOString(), + ), ); const seeded = await readDeploymentIdentityProtocol(execute); if (seeded === undefined) { @@ -260,4 +418,10 @@ export async function provisionDeploymentIdentityProtocol( if (seeded !== tag) { throw differentOwnerError(caller, seeded, tag); } + // Last, never first: the fence DDL is the one statement that could add a + // table to an as-yet-unowned database, and running it only after ownership is + // PROVEN keeps it out of the window where `unownedDatabaseError` and the + // conditional ownership insert are still deciding whether this database is + // ours to write to at all. + await seedExecutionFence(execute, fenceState, seededAt); } diff --git a/packages/flowsafe/examples/gtm-outbound.e2e.test.ts b/packages/flowsafe/examples/gtm-outbound.e2e.test.ts index 8b9cc21..9df05cf 100644 --- a/packages/flowsafe/examples/gtm-outbound.e2e.test.ts +++ b/packages/flowsafe/examples/gtm-outbound.e2e.test.ts @@ -82,7 +82,11 @@ function buildHarness(): Harness { // mints requestContext from approved records on every start/resume. const { createWorkflow, createStep, runtime } = init( { storage: new InMemoryStore() }, - { requestContextForRun: approvalGrantProvider(store) }, + { + startIdempotency: 'none', + requestContextForRun: approvalGrantProvider(store), + executionFence: 'none', + }, ); const researchAccounts = createStep({ @@ -185,6 +189,9 @@ function buildHarness(): Harness { const service = new ApprovalService({ store, + // In-memory store, no database to fence against: the opt-out is written down + // rather than defaulted — see ExecutionFenceWiring. + executionFence: 'none', resumeRun: resumeViaRuntime(runtime), }); return { runtime, service, sends: () => sends, audit }; diff --git a/packages/flowsafe/package.json b/packages/flowsafe/package.json index 4dc66da..bdad973 100644 --- a/packages/flowsafe/package.json +++ b/packages/flowsafe/package.json @@ -1,6 +1,6 @@ { "name": "@proofoftech/flowsafe", - "version": "0.19.0", + "version": "0.20.0", "publishConfig": { "access": "public" }, diff --git a/packages/flowsafe/scripts/agent-host-pack-test.mjs b/packages/flowsafe/scripts/agent-host-pack-test.mjs index 9c73759..907a493 100644 --- a/packages/flowsafe/scripts/agent-host-pack-test.mjs +++ b/packages/flowsafe/scripts/agent-host-pack-test.mjs @@ -204,12 +204,22 @@ import type { } from '@proofoftech/flowsafe/approval-api'; import { sweepExpiredRunDeadlines, + type DeploymentInventory, + type DrainProofContract, type DurableObjectRunLifecycleHooks, + type ExecutionFenceState, + type ExecutionFenceWiring, type RunTerminalErrorEnvelope, + type StartIdempotencyWiring, } from '@proofoftech/flowsafe/do-runner'; import { createFlowsafeRunnerLifecycle, createRunRouter, + type FlowsafeWorker, + type FlowsafeWorkerConfig, + type FlowsafeWorkerEnv, + type RunRouterOptions, + type RunRouterStartIdempotency, } from '@proofoftech/flowsafe/host-kit'; import { createAgentCatalog, @@ -221,6 +231,7 @@ import { type AgentAutomationRule, type AgentMeta, type AgentRunEnvelope, + type AgentThreadTopologyOptions, type AutomatedEntryAuthorizer, type AutomatedEntryRequest, type AutomationCheck, @@ -228,6 +239,10 @@ import { type PrincipalPermissionResolution, type PrincipalPermissionResolver, } from '@proofoftech/flowsafe/agent-host'; +import type { + BackgroundTaskHost, + BackgroundTaskReads, +} from '@proofoftech/flowsafe/background-tasks'; const automation: AgentAutomationRule = { kind: 'system', @@ -272,6 +287,21 @@ const terminalError: RunTerminalErrorEnvelope = { message: 'run was cancelled', }; const lifecycleHooks = null as DurableObjectRunLifecycleHooks | null; +const fenceState: ExecutionFenceState = 'open'; +const fenceWiring = null as ExecutionFenceWiring | null; +const startWiring = null as StartIdempotencyWiring | null; +const deploymentInventory = null as DeploymentInventory | null; +const drainProofReading = null as DrainProofContract['reading'] | null; +const routerOptions = null as RunRouterOptions | null; +const routerStart = null as RunRouterStartIdempotency | null; +const routerOptionStart = null as + | RunRouterOptions['startIdempotency'] + | null; +const workerConfig = null as FlowsafeWorkerConfig | null; +const worker = null as FlowsafeWorker | null; +const topologyOptions = null as AgentThreadTopologyOptions | null; +const backgroundReads = null as BackgroundTaskReads | null; +declare const bgHost: BackgroundTaskHost; void BREAKWATER_CONNECTOR_EXECUTION_KEY; void BREAKWATER_CONNECTOR_GRANTS_KEY; void connectorGrantsForLeg; @@ -288,6 +318,22 @@ void envelope; void grant; void terminalError; void lifecycleHooks; +void fenceState; +void fenceWiring; +void startWiring; +void deploymentInventory; +void drainProofReading; +void routerOptions; +void routerStart; +void routerOptionStart; +void workerConfig; +void worker; +void topologyOptions; +void backgroundReads; +void bgHost.enqueue; +void bgHost.getTask; +void bgHost.listTasks; +void bgHost.stream; void sweepExpiredRunDeadlines; void createFlowsafeRunnerLifecycle; void createRunRouter; @@ -314,18 +360,50 @@ void createRunRouter; import * as host from '@proofoftech/flowsafe/agent-host'; import * as flowsafe from '@proofoftech/flowsafe'; import * as approvals from '@proofoftech/flowsafe/approval-api'; -import { - createD1Storage, - sweepExpiredRunDeadlines, -} from '@proofoftech/flowsafe/do-runner'; -import { - createFlowsafeRunnerLifecycle, - createRunRouter, -} from '@proofoftech/flowsafe/host-kit'; -assert.equal(typeof createD1Storage, 'function'); -assert.equal(typeof sweepExpiredRunDeadlines, 'function'); -assert.equal(typeof createFlowsafeRunnerLifecycle, 'function'); -assert.equal(typeof createRunRouter, 'function'); +import * as backgroundTasks from '@proofoftech/flowsafe/background-tasks'; +import * as doRunner from '@proofoftech/flowsafe/do-runner'; +import * as hostKit from '@proofoftech/flowsafe/host-kit'; +for (const name of [ + 'createD1Storage', + 'sweepExpiredRunDeadlines', + 'ExecutionFenceStore', + 'executionFenceFor', + 'readExecutionFence', + 'ExecutionFencedError', + 'FenceTransitionConflictError', + 'ExecutionFenceUnreadableError', + 'admitsDrainableExecution', + 'admitsExistingRun', + 'admitsRunStart', + 'admitsWorkAuthoring', + 'StartIdempotencyStore', + 'startIdempotencyFor', + 'StartReservationOwnerMismatchError', + 'StartReservationTargetMismatchError', + 'IdempotentStartPendingError', + 'IdempotentStartUnresolvableError', + 'IdempotentStartAlreadySettledError', + 'DeploymentInventory', +]) { + assert.equal(typeof doRunner[name], 'function', name); +} +for (const name of [ + 'INVENTORY_CATEGORIES', + 'FLOWSAFE_TABLES', + 'INVENTORY_UNENUMERABLE', +]) { + assert.equal(Array.isArray(doRunner[name]), true, name); +} +assert.equal(typeof doRunner.INVENTORY_DRAIN_PROOF, 'object'); +assert.equal(typeof doRunner.INVENTORY_DRAIN_PROOF.reading, 'string'); +assert.equal(Array.isArray(doRunner.INVENTORY_DRAIN_PROOF.reachableFrom), true); +assert.equal(typeof hostKit.createFlowsafeRunnerLifecycle, 'function'); +assert.equal(typeof hostKit.createRunRouter, 'function'); +assert.equal(typeof hostKit.createFlowsafeWorker, 'function'); +assert.equal( + backgroundTasks.EXECUTION_FENCE_SUSPEND_KEY, + 'flowsafe.executionFenced', +); assert.equal(typeof host.createAgentCatalog, 'function'); assert.equal(typeof host.createAgentRouter, 'function'); assert.equal(typeof host.createAgentThreadTopology, 'function'); diff --git a/packages/flowsafe/scripts/attw-pack-check.mjs b/packages/flowsafe/scripts/attw-pack-check.mjs index ac0a56d..3f39d69 100644 --- a/packages/flowsafe/scripts/attw-pack-check.mjs +++ b/packages/flowsafe/scripts/attw-pack-check.mjs @@ -15,6 +15,27 @@ const IGNORED_PROFILE_PROBLEMS = new Map([ ['CJSResolvesToESM', 'node16-cjs'], ]); +/** + * Every emitted declaration that imports `#deployment-identity-protocol`, and + * therefore raises one accepted node10 internal-resolution error. + * + * Node 10 resolution predates subpath `imports`, so each of these is a + * declaration a Node 10 consumer cannot resolve. The package publishes esm-only + * and node10 is already an ignored profile, so they are accepted — but the list + * is deliberate, not a count: an internal import that genuinely does not + * resolve (a broken relative path in the emitted types) is exactly what this + * check exists to catch, and it would carry a different specifier or file. + * + * Adding a module that imports the protocol means adding it here. So does + * REMOVING one: the list is asserted EXHAUSTED below, so an entry that stops + * raising fails rather than sitting on as a stale pin nobody notices — which is + * how a list like this quietly stops describing the package it guards. + */ +export const PROTOCOL_IMPORTING_DECLARATIONS = [ + '/dist/do-runner/deployment-identity.d.ts', + '/dist/do-runner/execution-fence.d.ts', +]; + export function assertAttwEsmPackage(archive, cwd) { const attw = join(resolve(cwd, '..', '..'), 'node_modules', '.bin', 'attw'); const temporary = mkdtempSync(join(tmpdir(), 'flowsafe-attw-')); @@ -71,16 +92,50 @@ export function assertAttwEsmReport(report, status, stderr = '') { const internal = analysis.problems.filter( (problem) => problem.kind === 'InternalResolutionError', ); - assert.ok(internal.length <= 1, 'unexpected ATTW internal-resolution errors'); - if (internal[0]) { - assert.equal(internal[0].resolutionOption, 'node10'); - assert.equal(internal[0].moduleSpecifier, '#deployment-identity-protocol'); - assert.match( - internal[0].fileName, - /\/dist\/do-runner\/deployment-identity\.d\.ts$/, + // A duplicate entry would make the exhaustion check below unfalsifiable for + // the repeated file: one problem would satisfy both copies, so a genuinely + // stale second pin could never be detected. Rejected outright rather than + // deduplicated, because a list with a repeat in it was edited carelessly. + assert.equal( + new Set(PROTOCOL_IMPORTING_DECLARATIONS).size, + PROTOCOL_IMPORTING_DECLARATIONS.length, + `PROTOCOL_IMPORTING_DECLARATIONS has duplicate entries: ${JSON.stringify(PROTOCOL_IMPORTING_DECLARATIONS)}`, + ); + const matchedDeclarations = new Set(); + for (const problem of internal) { + const detail = JSON.stringify({ + fileName: problem.fileName, + moduleSpecifier: problem.moduleSpecifier, + resolutionOption: problem.resolutionOption, + }); + assert.equal( + problem.resolutionOption, + 'node10', + `unexpected ATTW internal-resolution error: ${detail}`, + ); + assert.equal( + problem.moduleSpecifier, + '#deployment-identity-protocol', + `unexpected ATTW internal-resolution error: ${detail}`, + ); + const matched = PROTOCOL_IMPORTING_DECLARATIONS.filter((suffix) => + problem.fileName.endsWith(suffix), ); + assert.ok( + matched.length > 0, + `unexpected ATTW internal-resolution error: ${detail}`, + ); + for (const suffix of matched) matchedDeclarations.add(suffix); } - assert.equal(status, internal.length, `ATTW exited ${status}\n${stderr}`); + // ATTW exits non-zero when it reports anything the profile does not ignore. + // Under esm-only the accepted node10 entries above are the only such + // problems, so a non-zero exit must be explained by them and a zero exit must + // mean there were none. + assert.equal( + status === 0, + internal.length === 0, + `ATTW exited ${status}\n${stderr}`, + ); for (const problem of analysis.problems) { if (problem.kind === 'InternalResolutionError') continue; @@ -94,4 +149,17 @@ export function assertAttwEsmReport(report, status, stderr = '') { `unexpected ATTW problem: ${JSON.stringify(problem)}`, ); } + + // LAST, so a report that fails one of the checks above says so instead of + // reporting the stale pins that failure implies. Every accepted entry must + // have been used: a module that stopped importing the protocol has stopped + // raising, and the entry that still names it is now a pin against nothing. + const stale = PROTOCOL_IMPORTING_DECLARATIONS.filter( + (suffix) => !matchedDeclarations.has(suffix), + ); + assert.deepEqual( + stale, + [], + `PROTOCOL_IMPORTING_DECLARATIONS names files that no longer raise a node10 internal-resolution error: ${JSON.stringify(stale)}`, + ); } diff --git a/packages/flowsafe/scripts/attw-pack-check.test.mjs b/packages/flowsafe/scripts/attw-pack-check.test.mjs index 2af381e..86137ef 100644 --- a/packages/flowsafe/scripts/attw-pack-check.test.mjs +++ b/packages/flowsafe/scripts/attw-pack-check.test.mjs @@ -1,5 +1,8 @@ import { describe, expect, it } from 'vitest'; -import { assertAttwEsmReport } from './attw-pack-check.mjs'; +import { + assertAttwEsmReport, + PROTOCOL_IMPORTING_DECLARATIONS, +} from './attw-pack-check.mjs'; function report(problems) { return { @@ -17,27 +20,78 @@ function report(problems) { }; } +function internalResolutionError(overrides = {}) { + return { + kind: 'InternalResolutionError', + resolutionOption: 'node10', + moduleSpecifier: '#deployment-identity-protocol', + fileName: + '/node_modules/@proofoftech/flowsafe/dist/do-runner/deployment-identity.d.ts', + ...overrides, + }; +} + describe('assertAttwEsmReport', () => { - it('accepts only the exact known Node10 internal-resolution exception', () => { + it('accepts every declaration that imports the protocol under Node10', () => { expect(() => assertAttwEsmReport( report([ - { - kind: 'InternalResolutionError', - resolutionOption: 'node10', - moduleSpecifier: '#deployment-identity-protocol', + internalResolutionError(), + internalResolutionError({ fileName: - '/node_modules/@proofoftech/flowsafe/dist/do-runner/deployment-identity.d.ts', - }, + '/node_modules/@proofoftech/flowsafe/dist/do-runner/execution-fence.d.ts', + }), ]), 1, ), ).not.toThrow(); }); + it('rejects an internal-resolution error that is not the known exception', () => { + // A genuinely broken relative import in the emitted types — the failure + // this check exists to catch — differs in specifier, file, or option. + for (const overrides of [ + { moduleSpecifier: './missing-module.js' }, + { resolutionOption: 'node16-esm' }, + { + fileName: + '/node_modules/@proofoftech/flowsafe/dist/host-kit/index.d.ts', + }, + ]) { + expect(() => + assertAttwEsmReport(report([internalResolutionError(overrides)]), 1), + ).toThrow(/unexpected ATTW internal-resolution error/); + } + }); + + it('rejects a zero exit that contradicts the reported problems', () => { + expect(() => + assertAttwEsmReport(report([internalResolutionError()]), 0), + ).toThrow(/ATTW exited 0/); + expect(() => assertAttwEsmReport(report([]), 1)).toThrow(/ATTW exited 1/); + }); + it('rejects unknown problem kinds even when their resolution is absent', () => { expect(() => assertAttwEsmReport(report([{ kind: 'FalseESM' }]), 0), ).toThrow(/unexpected ATTW problem/); }); + + it('rejects a pinned declaration that no longer raises the accepted error', () => { + // #given — the protocol-importing list is an ACCEPT list, so a stale entry + // silently widens what the check tolerates: the day a module stops + // importing the protocol, its entry keeps admitting a future error at the + // same path. Only one of the two declarations reports. + expect(() => + assertAttwEsmReport(report([internalResolutionError()]), 1), + ).toThrow(/no longer raise a node10 internal-resolution error/); + }); + + it('keeps the accepted declaration list free of duplicates', () => { + // #given — a repeat would make the exhaustion check above unfalsifiable for + // that file: one problem would satisfy both copies. + expect(new Set(PROTOCOL_IMPORTING_DECLARATIONS).size).toBe( + PROTOCOL_IMPORTING_DECLARATIONS.length, + ); + }); }); diff --git a/packages/flowsafe/scripts/provisioning-pack-test.mjs b/packages/flowsafe/scripts/provisioning-pack-test.mjs index fc545fd..b724a68 100644 --- a/packages/flowsafe/scripts/provisioning-pack-test.mjs +++ b/packages/flowsafe/scripts/provisioning-pack-test.mjs @@ -70,17 +70,33 @@ if (typeof sql !== 'string') { const statePath = process.env.FAKE_WRANGLER_STATE; const state = existsSync(statePath) ? JSON.parse(readFileSync(statePath, 'utf8')) - : { created: false, tag: undefined }; + : { created: false, tag: undefined, fence: false, fenceState: undefined }; +const FENCE = 'flowsafe_execution_fence'; const schema = \`CREATE TABLE flowsafe_deployment ( id INTEGER PRIMARY KEY CHECK (id = 1), tenant_tag TEXT NOT NULL, provisioned_at TEXT NOT NULL )\`; let results; +// Fence statements are matched by their TARGET table and BEFORE the generic +// arms: the ownership insert names the fence table inside its exclusion list, +// so a substring test would route it to the wrong branch. if (sql.startsWith('SELECT name, sql')) { - results = state.created - ? [{ name: 'flowsafe_deployment', sql: schema }] - : []; + results = [ + ...(state.created ? [{ name: 'flowsafe_deployment', sql: schema }] : []), + ...(state.fence ? [{ name: FENCE, sql: 'CREATE' }] : []), + ]; +} else if (sql.startsWith('CREATE TABLE IF NOT EXISTS ' + FENCE)) { + state.fence = true; + results = []; +} else if (sql.startsWith('INSERT OR IGNORE INTO ' + FENCE)) { + if (!state.fence) { + process.stderr.write('fence row seeded before its table\\n'); + process.exit(5); + } + state.fenceState = + state.fenceState ?? sql.match(/'deployment', '([^']+)'/)?.[1]; + results = []; } else if (sql.startsWith('CREATE TABLE')) { state.created = true; results = []; @@ -215,10 +231,28 @@ writeFileSync(new URL('./install-script-ran', import.meta.url), 'unexpected'); `import assert from 'node:assert/strict'; import { DEPLOYMENT_IDENTITY_HEADER, + EXECUTION_FENCE_DDL, + EXECUTION_FENCE_ROW_ID, + EXECUTION_FENCE_STATES, + EXECUTION_FENCE_TABLE, + INITIAL_EXECUTION_FENCE_STATES, deploymentIdentityHeaders, } from '@proofoftech/flowsafe/deployment-identity-protocol'; const secret = 'x'.repeat(32); +assert.equal(typeof EXECUTION_FENCE_DDL, 'string'); +assert.equal(EXECUTION_FENCE_ROW_ID, 'deployment'); +assert.deepEqual(EXECUTION_FENCE_STATES, [ + 'open', + 'draining', + 'migration-locked', + 'proof-only', +]); +assert.equal(EXECUTION_FENCE_TABLE, 'flowsafe_execution_fence'); +assert.deepEqual(INITIAL_EXECUTION_FENCE_STATES, [ + 'open', + 'migration-locked', +]); assert.deepEqual( deploymentIdentityHeaders(secret, { 'content-type': 'application/json', @@ -236,6 +270,7 @@ assert.deepEqual( `import { DEPLOYMENT_IDENTITY_HEADER, deploymentIdentityHeaders, + type InitialExecutionFenceState, } from '@proofoftech/flowsafe/deployment-identity-protocol'; import { DEPLOYMENT_IDENTITY_HEADER as LEGACY_DEPLOYMENT_IDENTITY_HEADER, @@ -248,9 +283,11 @@ const legacyHeaders: Record = legacyDeploymentIdentityHeaders(secret); const header: typeof DEPLOYMENT_IDENTITY_HEADER = LEGACY_DEPLOYMENT_IDENTITY_HEADER; +const initialFenceState: InitialExecutionFenceState = 'open'; void headers; void legacyHeaders; void header; +void initialFenceState; `, ); writeFileSync( @@ -322,6 +359,19 @@ void header; 'packed package is missing the deployment identity protocol implementation or declaration', ); } + // The fence schema ships in the SAME root module the runtime store imports + // it from; if it were missing here, the packed store would fail to resolve + // its own DDL rather than silently create a different table. + if ( + !protocolSource.includes('EXECUTION_FENCE_DDL') || + !protocolSource.includes('assertInitialExecutionFenceState') || + !protocolDeclaration.includes('InitialExecutionFenceState') || + !protocolDeclaration.includes('EXECUTION_FENCE_DDL') + ) { + throw new Error( + 'packed deployment identity protocol is missing the execution fence seeding contract', + ); + } const runtimeIdentity = await import( pathToFileURL( join(packageRoot, 'dist', 'do-runner', 'deployment-identity.js'), @@ -390,11 +440,48 @@ void header; const statePath = join(consumerRoot, 'wrangler-state.json'); const logPath = join(consumerRoot, 'wrangler-invocations.ndjson'); writeFileSync(logPath, ''); + // The fence state has NO default, so an otherwise-complete invocation that + // omits it must fail rather than quietly provision an executing deployment. + const withoutFenceState = invokeProvision(consumerRoot, [ + '--database', + 'consumer-db', + '--tag', + 'acme', + '--local', + ]); + if ( + withoutFenceState.status !== 1 || + !withoutFenceState.stderr.includes('--initial-fence-state') + ) { + throw new Error( + `packed provisioning CLI accepted a missing --initial-fence-state (status=${withoutFenceState.status})\n${withoutFenceState.stdout}\n${withoutFenceState.stderr}`, + ); + } + const badFenceState = invokeProvision(consumerRoot, [ + '--database', + 'consumer-db', + '--tag', + 'acme', + '--initial-fence-state', + 'draining', + '--local', + ]); + if ( + badFenceState.status !== 1 || + !badFenceState.stderr.includes('must be one of open, migration-locked') + ) { + throw new Error( + `packed provisioning CLI accepted a non-birth fence state (status=${badFenceState.status})\n${badFenceState.stdout}\n${badFenceState.stderr}`, + ); + } + const validArgs = [ '--database', 'consumer-db', '--tag', 'acme', + '--initial-fence-state', + 'migration-locked', '--local', '--config', 'wrangler.jsonc', @@ -408,7 +495,7 @@ void header; if ( valid.status !== 0 || valid.stdout !== - "Deployment identity 'acme' verified in consumer-db (local).\n" || + "Deployment identity 'acme' verified in consumer-db (local), initial execution fence state 'migration-locked'.\n" || valid.stderr !== '' ) { throw new Error( @@ -425,6 +512,36 @@ void header; `expected full provisioning query sequence, got ${invocations.length}`, ); } + // The SQL follows `--command`; `--config`/`--persist-to` come after it, so + // the last argument is not the statement. + const executedSql = invocations.map( + (invocation) => invocation.args[invocation.args.indexOf('--command') + 1], + ); + const fenceDdlAt = executedSql.findIndex((sql) => + sql.startsWith('CREATE TABLE IF NOT EXISTS flowsafe_execution_fence'), + ); + const fenceRowAt = executedSql.findIndex((sql) => + sql.startsWith('INSERT OR IGNORE INTO flowsafe_execution_fence'), + ); + const ownershipAt = executedSql.findIndex((sql) => + sql.startsWith('INSERT OR IGNORE INTO flowsafe_deployment'), + ); + if ( + fenceDdlAt === -1 || + fenceRowAt !== fenceDdlAt + 1 || + ownershipAt === -1 || + ownershipAt > fenceDdlAt + ) { + throw new Error( + `packed provisioning CLI did not seed the fence after proving ownership: ${JSON.stringify(executedSql)}`, + ); + } + const seededState = JSON.parse(readFileSync(statePath, 'utf8')).fenceState; + if (seededState !== 'migration-locked') { + throw new Error( + `packed provisioning CLI seeded fence state '${seededState}', expected 'migration-locked'`, + ); + } for (const invocation of invocations) { const expectedPrefix = [ 'd1', @@ -456,6 +573,8 @@ void header; 'consumer-db', '--tag', 'acme', + '--initial-fence-state', + 'open', '--preview', '--config', 'wrangler.jsonc', @@ -467,7 +586,7 @@ void header; if ( preview.status !== 0 || preview.stdout !== - "Deployment identity 'acme' verified in consumer-db (preview).\n" || + "Deployment identity 'acme' verified in consumer-db (preview), initial execution fence state 'open'.\n" || preview.stderr !== '' ) { throw new Error( diff --git a/packages/flowsafe/scripts/seed-deployment-identity.mjs b/packages/flowsafe/scripts/seed-deployment-identity.mjs index c142e0f..684e6b5 100755 --- a/packages/flowsafe/scripts/seed-deployment-identity.mjs +++ b/packages/flowsafe/scripts/seed-deployment-identity.mjs @@ -9,13 +9,14 @@ import { fileURLToPath } from 'node:url'; import { DEPLOYMENT_TAG_PATTERN, + INITIAL_EXECUTION_FENCE_STATES, provisionDeploymentIdentityProtocol, } from '#deployment-identity-protocol'; function usage(message) { const suffix = message ? `\n${message}` : ''; return new Error( - `Usage: flowsafe-provision --database --tag (--remote | --local | --preview) [--config ] [--persist-to ]${suffix}`, + `Usage: flowsafe-provision --database --tag --initial-fence-state <${INITIAL_EXECUTION_FENCE_STATES.join('|')}> (--remote | --local | --preview) [--config ] [--persist-to ]${suffix}`, ); } @@ -23,6 +24,7 @@ export function parseProvisioningArguments(argv) { const valueOptions = new Set([ '--database', '--tag', + '--initial-fence-state', '--config', '--persist-to', ]); @@ -48,10 +50,19 @@ export function parseProvisioningArguments(argv) { } const database = values.get('--database'); const tag = values.get('--tag'); - if (!database || !tag || !target) throw usage(); + // No default, deliberately: an operator provisioning a deployment that is + // about to receive a migration must state that it starts locked, and one who + // says nothing must be told rather than handed an executing deployment. + const initialFenceState = values.get('--initial-fence-state'); + if (!database || !tag || !target || !initialFenceState) throw usage(); if (!DEPLOYMENT_TAG_PATTERN.test(tag)) { throw usage(`tag '${tag}' must match ${DEPLOYMENT_TAG_PATTERN}`); } + if (!INITIAL_EXECUTION_FENCE_STATES.includes(initialFenceState)) { + throw usage( + `--initial-fence-state '${initialFenceState}' must be one of ${INITIAL_EXECUTION_FENCE_STATES.join(', ')}`, + ); + } const persistTo = values.get('--persist-to'); if (persistTo && target !== '--local') { throw usage('--persist-to is valid only with --local'); @@ -60,6 +71,7 @@ export function parseProvisioningArguments(argv) { database, tag, target, + initialFenceState, ...(values.has('--config') ? { config: values.get('--config') } : {}), ...(persistTo ? { persistTo } : {}), }; @@ -235,15 +247,21 @@ export async function provisionDeploymentIdentity(options, execute) { await provisionDeploymentIdentityProtocol( (statement) => query(renderProtocolStatement(statement)), options.tag, - { caller: 'flowsafe-provision' }, + { + caller: 'flowsafe-provision', + initialExecutionFenceState: options.initialFenceState, + }, ); } async function main() { const options = parseProvisioningArguments(process.argv.slice(2)); await provisionDeploymentIdentity(options); + // The fence line names the state that was REQUESTED, not one that was + // necessarily written: seeding is INSERT-if-absent, so a database whose fence + // an operator already moved keeps the state it has. process.stdout.write( - `Deployment identity '${options.tag}' verified in ${options.database} (${options.target.slice(2)}).\n`, + `Deployment identity '${options.tag}' verified in ${options.database} (${options.target.slice(2)}), initial execution fence state '${options.initialFenceState}'.\n`, ); } diff --git a/packages/flowsafe/scripts/seed-deployment-identity.test.mjs b/packages/flowsafe/scripts/seed-deployment-identity.test.mjs index 39f9b2f..5dffe01 100644 --- a/packages/flowsafe/scripts/seed-deployment-identity.test.mjs +++ b/packages/flowsafe/scripts/seed-deployment-identity.test.mjs @@ -18,7 +18,9 @@ const OPTIONS = { database: 'flowsafe-acme', tag: 'acme', target: '--remote', + initialFenceState: 'open', }; +const FENCE_TABLE = 'flowsafe_execution_fence'; const SQL = `CREATE TABLE flowsafe_deployment ( id INTEGER PRIMARY KEY CHECK (id = 1), tenant_tag TEXT NOT NULL, @@ -34,29 +36,52 @@ function databaseQuery(initialTables = [], ownerTag = 'acme') { const tables = [...initialTables]; let seeded = initialTables.some((row) => row.name === 'flowsafe_deployment'); let storedOwner = seeded ? (ownerTag ?? undefined) : undefined; + // The fence table is reported by the schema scan once created, so the + // ownership guard below faces the same residue a crashed provisioning pass + // would leave behind. + let fenceTable = initialTables.some((row) => row.name === FENCE_TABLE); + let fenceState; const mutations = []; return { mutations, + fence: () => ({ table: fenceTable, state: fenceState }), addTable: (row) => tables.push(row), query: async (statement) => { if (statement.startsWith('SELECT name, sql')) { const applicationTables = tables.filter( - (row) => row.name !== 'flowsafe_deployment', + (row) => + row.name !== 'flowsafe_deployment' && row.name !== FENCE_TABLE, ); - return seeded - ? [{ name: 'flowsafe_deployment', sql: SQL }, ...applicationTables] - : applicationTables; + return [ + ...(seeded ? [{ name: 'flowsafe_deployment', sql: SQL }] : []), + ...(fenceTable ? [{ name: FENCE_TABLE, sql: 'CREATE' }] : []), + ...applicationTables, + ]; } if (statement.startsWith('CREATE TABLE')) { mutations.push(statement); - seeded = true; + // Dispatch on the TARGET table, never on a substring: the ownership + // insert names the fence table in its exclusion list, so `includes` + // would route it here. + if (statement.startsWith(`CREATE TABLE IF NOT EXISTS ${FENCE_TABLE}`)) { + fenceTable = true; + } else { + seeded = true; + } return []; } if (statement.startsWith('INSERT OR IGNORE')) { mutations.push(statement); + if (statement.startsWith(`INSERT OR IGNORE INTO ${FENCE_TABLE}`)) { + // INSERT OR IGNORE: an existing row wins, exactly as the protocol + // requires so a re-provision cannot reopen a closed fence. + fenceState ??= statement.match(/VALUES \('[^']+', '([^']+)'/)?.[1]; + return []; + } const blocking = tables.filter( (row) => row.name !== 'flowsafe_deployment' && + row.name !== FENCE_TABLE && row.name !== '_cf_KV' && row.name !== '_cf_METADATA' && !row.name.startsWith('sqlite_'), @@ -108,6 +133,29 @@ function sentinelSnapshot(sqlite) { }; } +// The fence table AS SQLITE STORED IT, plus its row without the timestamp. +// This is what proves the runtime store and the provisioning protocol issue one +// schema: a drifted copy would still be accepted by `CREATE TABLE IF NOT +// EXISTS`, so only comparing the materialized tables catches it. +function fenceSnapshot(sqlite) { + const schema = sqlite + .prepare( + `SELECT sql FROM sqlite_schema WHERE type = 'table' AND name = '${FENCE_TABLE}'`, + ) + .get(); + return { + sql: schema.sql.replace(/\s+/g, ' ').trim(), + columns: sqlite.prepare(`PRAGMA table_info(${FENCE_TABLE})`).all(), + rows: sqlite + .prepare(`SELECT id, state, proof_key, proof_run_id FROM ${FENCE_TABLE}`) + .all(), + timestamps: sqlite + .prepare(`SELECT updated_at FROM ${FENCE_TABLE}`) + .all() + .map((row) => typeof row.updated_at), + }; +} + async function rejectedError(action) { try { await action(); @@ -118,7 +166,7 @@ async function rejectedError(action) { } describe('deployment identity provisioning CLI', () => { - it('requires an explicit database, valid tag, and execution target', () => { + it('requires an explicit database, valid tag, fence state, and execution target', () => { expect( parseProvisioningArguments([ '--', @@ -126,6 +174,8 @@ describe('deployment identity provisioning CLI', () => { 'flowsafe-acme', '--tag', 'acme', + '--initial-fence-state', + 'open', '--remote', ]), ).toEqual(OPTIONS); @@ -135,6 +185,8 @@ describe('deployment identity provisioning CLI', () => { 'flowsafe-acme', '--tag', 'ACME', + '--initial-fence-state', + 'open', '--remote', ]), ).toThrow(/must match/); @@ -144,6 +196,8 @@ describe('deployment identity provisioning CLI', () => { 'flowsafe-acme', '--tag', 'acme', + '--initial-fence-state', + 'open', ]), ).toThrow(/Usage/); expect(() => @@ -152,12 +206,49 @@ describe('deployment identity provisioning CLI', () => { 'flowsafe-acme', '--tag', 'acme', + '--initial-fence-state', + 'open', '--remote', '--preview', ]), ).toThrow(/choose exactly one execution target/); }); + it('refuses a missing or unknown initial fence state', () => { + // No default: omitting the flag is a usage error, not an open deployment. + expect(() => + parseProvisioningArguments([ + '--database', + 'flowsafe-acme', + '--tag', + 'acme', + '--remote', + ]), + ).toThrow(/Usage/); + for (const state of ['draining', 'proof-only', 'OPEN', 'locked']) { + expect(() => + parseProvisioningArguments([ + '--database', + 'flowsafe-acme', + '--tag', + 'acme', + '--initial-fence-state', + state, + '--remote', + ]), + ).toThrow(/must be one of open, migration-locked/); + } + }); + + it('accepts migration-locked and seeds it verbatim', async () => { + const fake = databaseQuery(); + await provisionDeploymentIdentity( + { ...OPTIONS, initialFenceState: 'migration-locked' }, + fake.query, + ); + expect(fake.fence()).toEqual({ table: true, state: 'migration-locked' }); + }); + it.each([ ['--local', ['--local']], ['--remote', ['--remote']], @@ -183,9 +274,63 @@ describe('deployment identity provisioning CLI', () => { it('seeds a fresh database and verifies the strict sentinel', async () => { const fake = databaseQuery(); await provisionDeploymentIdentity(OPTIONS, fake.query); - expect(fake.mutations).toHaveLength(2); - expect(fake.mutations[0]).toMatch(/^CREATE TABLE/); - expect(fake.mutations[1]).toMatch(/^INSERT OR IGNORE/); + // Sentinel DDL, ownership insert, then the fence: the fence DDL runs LAST + // so it can never add a table to a database whose ownership is still being + // decided. + expect(fake.mutations).toHaveLength(4); + expect(fake.mutations[0]).toMatch( + /^CREATE TABLE IF NOT EXISTS flowsafe_deployment/, + ); + expect(fake.mutations[1]).toMatch( + /^INSERT OR IGNORE INTO flowsafe_deployment/, + ); + expect(fake.mutations[2]).toMatch( + /^CREATE TABLE IF NOT EXISTS flowsafe_execution_fence/, + ); + expect(fake.mutations[3]).toMatch( + /^INSERT OR IGNORE INTO flowsafe_execution_fence/, + ); + expect(fake.fence()).toEqual({ table: true, state: 'open' }); + }); + + it('re-seeds the fence on the already-owned early return without reopening it', async () => { + const fake = databaseQuery(); + await provisionDeploymentIdentity(OPTIONS, fake.query); + const afterFirst = fake.mutations.length; + + // A second pass short-circuits on ownership but still writes the fence, so + // a run that died between the ownership insert and the fence row heals. + await provisionDeploymentIdentity( + { ...OPTIONS, initialFenceState: 'migration-locked' }, + fake.query, + ); + + expect(fake.mutations.slice(afterFirst)).toHaveLength(2); + expect(fake.mutations[afterFirst]).toMatch( + /^CREATE TABLE IF NOT EXISTS flowsafe_execution_fence/, + ); + expect(fake.mutations[afterFirst + 1]).toMatch( + /^INSERT OR IGNORE INTO flowsafe_execution_fence/, + ); + // INSERT-if-absent: the existing row survives a re-provision that asked for + // a different state. + expect(fake.fence()).toEqual({ table: true, state: 'open' }); + }); + + it('heals a fence row that a previous pass never wrote', async () => { + // The residue of a crash between the ownership insert and the fence seed: + // an owned database whose fence table exists but holds no row. + const fake = databaseQuery([ + { name: 'flowsafe_deployment', sql: SQL }, + { name: FENCE_TABLE, sql: 'CREATE' }, + ]); + + await provisionDeploymentIdentity( + { ...OPTIONS, initialFenceState: 'migration-locked' }, + fake.query, + ); + + expect(fake.fence()).toEqual({ table: true, state: 'migration-locked' }); }); it('seeds the exact schema accepted by the runtime guard', async () => { @@ -200,14 +345,35 @@ describe('deployment identity provisioning CLI', () => { it('keeps the runtime and packed-CLI adapters on the same protocol schema and idempotency path', async () => { const runtimeSqlite = openSqlite(); const cliSqlite = openSqlite(); - await seedDeploymentIdentity(sqliteUnitDatabase(runtimeSqlite), 'acme'); + await seedDeploymentIdentity( + sqliteUnitDatabase(runtimeSqlite), + 'acme', + 'open', + ); await provisionDeploymentIdentity(OPTIONS, sqliteQuery(cliSqlite)); expect(sentinelSnapshot(runtimeSqlite)).toEqual( sentinelSnapshot(cliSqlite), ); + // The fence table too: the runtime binds its parameters and the CLI renders + // them as literals, so agreeing here is what proves one DDL and one row + // shape reach D1 down both paths. + expect(fenceSnapshot(runtimeSqlite)).toEqual(fenceSnapshot(cliSqlite)); + expect(fenceSnapshot(runtimeSqlite)).toMatchObject({ + rows: [ + { + id: 'deployment', + state: 'open', + proof_key: null, + proof_run_id: null, + }, + ], + // Bound as a string against an INTEGER column: SQLite's affinity has to + // have converted it, or every later `updated_at` comparison is text. + timestamps: ['number'], + }); await provisionDeploymentIdentity(OPTIONS, sqliteQuery(runtimeSqlite)); - await seedDeploymentIdentity(sqliteUnitDatabase(cliSqlite), 'acme'); + await seedDeploymentIdentity(sqliteUnitDatabase(cliSqlite), 'acme', 'open'); expect( await readDeploymentIdentity(sqliteUnitDatabase(runtimeSqlite)), ).toBe('acme'); @@ -235,7 +401,7 @@ describe('deployment identity provisioning CLI', () => { cliSqlite.exec(ddl); const runtimeError = await rejectedError(() => - seedDeploymentIdentity(sqliteUnitDatabase(runtimeSqlite), 'acme'), + seedDeploymentIdentity(sqliteUnitDatabase(runtimeSqlite), 'acme', 'open'), ); const cliError = await rejectedError(() => provisionDeploymentIdentity(OPTIONS, sqliteQuery(cliSqlite)), @@ -253,8 +419,16 @@ describe('deployment identity provisioning CLI', () => { await provisionDeploymentIdentity(OPTIONS, fake.query); - expect(fake.mutations).toHaveLength(1); - expect(fake.mutations[0]).toMatch(/^INSERT OR IGNORE/); + expect(fake.mutations).toHaveLength(3); + expect(fake.mutations[0]).toMatch( + /^INSERT OR IGNORE INTO flowsafe_deployment/, + ); + expect(fake.mutations[1]).toMatch( + /^CREATE TABLE IF NOT EXISTS flowsafe_execution_fence/, + ); + expect(fake.mutations[2]).toMatch( + /^INSERT OR IGNORE INTO flowsafe_execution_fence/, + ); }); it('refuses an unowned database with any application table', async () => { @@ -283,7 +457,19 @@ describe('deployment identity provisioning CLI', () => { ])('allows the exact D1-owned %s table', async (name) => { const fake = databaseQuery([{ name, sql: 'CREATE' }]); await provisionDeploymentIdentity(OPTIONS, fake.query); - expect(fake.mutations).toHaveLength(2); + expect(fake.mutations).toHaveLength(4); + }); + + it('allows a pre-existing execution fence table left by an interrupted pass', async () => { + // The fence table is protocol-owned, so its presence must never be read as + // "somebody else's application data" — otherwise a pass that died after the + // fence DDL would refuse the database it had just started provisioning. + const fake = databaseQuery([{ name: FENCE_TABLE, sql: 'CREATE' }]); + await provisionDeploymentIdentity(OPTIONS, fake.query); + expect(await fake.query('SELECT id FROM flowsafe_deployment')).toEqual([ + { id: 1, tenant_tag: 'acme' }, + ]); + expect(fake.fence()).toEqual({ table: true, state: 'open' }); }); it('refuses a table created between scan and the conditional insert', async () => { @@ -337,7 +523,7 @@ describe('deployment identity provisioning CLI', () => { }); const runtimeError = await rejectedError(() => - seedDeploymentIdentity(racingRuntime, 'acme'), + seedDeploymentIdentity(racingRuntime, 'acme', 'open'), ); const cliError = await rejectedError(() => provisionDeploymentIdentity(OPTIONS, racingCliQuery), diff --git a/packages/flowsafe/scripts/spike-verify.mjs b/packages/flowsafe/scripts/spike-verify.mjs index 2aedfc5..98ec449 100644 --- a/packages/flowsafe/scripts/spike-verify.mjs +++ b/packages/flowsafe/scripts/spike-verify.mjs @@ -33,6 +33,11 @@ import { tmpdir } from 'node:os'; import { dirname, join } from 'node:path'; import { fileURLToPath } from 'node:url'; import { SignJWT } from 'jose'; +import { + EXECUTION_FENCE_DDL, + EXECUTION_FENCE_ROW_ID, + EXECUTION_FENCE_TABLE, +} from '#deployment-identity-protocol'; import { createWorkerdServerLifecycle, parsePort, @@ -65,6 +70,9 @@ const AGENT_RUN_BODY = { // and still elapse inside the kill+restart window, so it adds no wall clock of // its own; hence 10s rather than the module's 1s floor. The SIGNAL deadline // only has to outlive its own immediate resume, which settles the entry. +// Idempotent start (FI1/FI2): demo-approval's shape with a counting first step +// (spike/worker.ts COUNTED_WORKFLOW_ID). MUST match the worker's id. +const COUNTED_WORKFLOW_ID = 'demo-idempotent'; const DEADLINE_WORKFLOW_ID = 'demo-deadline'; const DEADLINE_STEP = 'wait-signal'; // The reserved arming key, as it appears on the wire (do-runner exports it to @@ -329,6 +337,156 @@ function deadlineRunPath(runId) { return `/runs/${DEADLINE_WORKFLOW_ID}/${encodeURIComponent(runId)}`; } +// --- Execution fence (F1) helpers ------------------------------------------ +// The fence control channel and the two assertions every fence probe makes: +// the exact refusal CODE (not merely "some 5xx"), and the state it names. + +const FENCE_ADMIN_PATH = '/admin/execution-fence'; + +async function readFence() { + const { status, body } = await http('GET', FENCE_ADMIN_PATH); + assert(status === 200, `GET ${FENCE_ADMIN_PATH} -> ${status}`, body); + return body; +} + +async function moveFence(expected, next, proofKey) { + const { status, body } = await http('POST', FENCE_ADMIN_PATH, { + body: { expected, next, ...(proofKey === undefined ? {} : { proofKey }) }, + }); + assert( + status === 200 && body.state === next, + `fence CAS ${expected} -> ${next} failed`, + { status, body }, + ); + return body; +} + +// A keyed start, as a trusted client actually sends one: the key rides the +// PUBLIC body (it names a request, unlike the runId, which stays server-minted +// and is still refused with a 400 here). +async function startWithKey(idempotencyKey, headers = AUTH.operator) { + return http('POST', '/runs', { + body: { ...RUN_BODY, idempotencyKey }, + headers, + }); +} + +// The same keyed start against the COUNTING workflow, whose first step +// increments a durable D1 row before the run suspends. `counterId` names the +// row, so each probe counts only its own executions. +async function startCountedWithKey( + idempotencyKey, + counterId, + headers = AUTH.operator, +) { + return http('POST', '/runs', { + body: { + workflowId: COUNTED_WORKFLOW_ID, + inputData: { topic: 'launch', counterId }, + idempotencyKey, + }, + headers, + }); +} + +// How many times the counting step actually ran for this counter. THE +// assertion FI1 and FI2 are really about: a repeated run id says two responses +// named one run, while this says the paid work happened once. +async function executionCount(counterId) { + const { status, body } = await http( + 'GET', + `/idempotent/executions?counterId=${encodeURIComponent(counterId)}`, + ); + assert(status === 200, `execution-count probe -> ${status}`, body); + return body.executions; +} + +// Every fence refusal is 503 (operator-transient, retryable) carrying +// reason.code EXECUTION_FENCED and the state that refused. Asserting the code +// and the state — rather than the status alone — is what separates "the fence +// refused" from "something else broke with a 5xx". +function assertFenced(label, response, state) { + assert( + response.status === 503 && + response.body?.reason?.code === 'EXECUTION_FENCED' && + response.body?.reason?.state === state, + `${label} must be refused 503 EXECUTION_FENCED in '${state}'`, + response, + ); +} + +// --- Drain inventory (F2) helpers ------------------------------------------ + +const INVENTORY_ADMIN_PATH = '/admin/inventory'; + +async function readInventoryIndex() { + const { status, body } = await http('GET', INVENTORY_ADMIN_PATH); + assert(status === 200, `GET ${INVENTORY_ADMIN_PATH} -> ${status}`, body); + return body; +} + +// One category, PAGED TO EXHAUSTION through the inventory's own cursors — the +// sweep an operator's drain proof is actually made of, not a single page that +// could hide the rows behind it. `limit: 3` forces several continuations even +// on the spike's small data, so the keyset is exercised rather than skipped. +async function sweepInventoryCategory(category) { + const keys = []; + let first; + let cursor; + for (let pass = 0; pass < 200; pass += 1) { + const query = `${INVENTORY_ADMIN_PATH}?category=${encodeURIComponent(category)}&limit=3${ + cursor === undefined ? '' : `&cursor=${encodeURIComponent(cursor)}` + }`; + const { status, body } = await http('GET', query); + assert(status === 200, `GET ${query} -> ${status}`, body); + first ??= body; + for (const entry of body.entries) keys.push(entry.key.join('/')); + if (body.cursor === undefined) { + return { + keys, + count: first.count, + totals: first.totals, + class: body.class, + table: body.table, + }; + } + cursor = body.cursor; + } + throw new Error(`inventory paging did not terminate for ${category}`); +} + +// Every WORK category, swept. The drain proof is defined over exactly these. +async function sweepInventoryWork(index) { + const work = index.categories.filter((entry) => entry.class === 'work'); + const swept = {}; + for (const entry of work) { + swept[entry.category] = await sweepInventoryCategory(entry.category); + } + return swept; +} + +async function startFencedRun(headers = AUTH.operator) { + const { status, body } = await http('POST', '/runs', { + body: RUN_BODY, + headers, + }); + assert( + status === 200 && body.status === 'suspended', + `fence-probe run did not suspend -> ${status}`, + body, + ); + assert( + typeof body.approval?.id === 'string', + 'fence-probe approval id', + body, + ); + return { + runId: body.runId, + approvalId: body.approval.id, + step: body.suspended?.[0], + }; +} + // --- Track E (M-007) webhook probe helpers --------------------------------- // GitHub's X-Hub-Signature-256 = 'sha256=' + hex HMAC-SHA256(secret, rawBody). @@ -575,10 +733,21 @@ async function main() { tmpDir = mkdtempSync(join(tmpdir(), 'spike-verify-')); const stateDir = join(tmpDir, 'state'); - await step('provision deployment identity sentinel', () => + // Provisioning, as the protocol performs it: the ownership sentinel AND an + // EXPLICIT execution fence row. The fence DDL is the protocol's own constant + // (the same string do-runner/execution-fence.ts issues), so this database is + // shaped exactly like one flowsafe-provision would have produced — a + // hand-copied schema here would let the store's CREATE TABLE IF NOT EXISTS + // silently accept a table with no CHECK constraints. 'open' because every + // scenario before the fence probes needs an executing deployment. + await step('provision deployment identity sentinel and execution fence', () => executeLocalD1( stateDir, - "CREATE TABLE IF NOT EXISTS flowsafe_deployment (id INTEGER PRIMARY KEY CHECK (id = 1), tenant_tag TEXT NOT NULL, provisioned_at TEXT NOT NULL); INSERT OR IGNORE INTO flowsafe_deployment (id, tenant_tag, provisioned_at) VALUES (1, 'spike', datetime('now'));", + 'CREATE TABLE IF NOT EXISTS flowsafe_deployment (id INTEGER PRIMARY KEY CHECK (id = 1), tenant_tag TEXT NOT NULL, provisioned_at TEXT NOT NULL); ' + + "INSERT OR IGNORE INTO flowsafe_deployment (id, tenant_tag, provisioned_at) VALUES (1, 'spike', datetime('now')); " + + `${EXECUTION_FENCE_DDL}; ` + + `INSERT OR IGNORE INTO ${EXECUTION_FENCE_TABLE} (id, state, proof_key, proof_run_id, updated_at) ` + + `VALUES ('${EXECUTION_FENCE_ROW_ID}', 'open', NULL, NULL, ${Date.now()});`, ), ); @@ -2311,6 +2480,600 @@ async function main() { }, ); + // --- Execution fence (F1): the migration control, on real workerd --------- + // Unit tests can prove the store's compare-and-set. What they cannot prove is + // that a fence written by one process still refuses work in the NEXT one, and + // that the refusal reaches an HTTP client as the taxonomy's own code rather + // than a generic 500 somewhere in the router chain. That is this scenario. + + const fenceRuns = await step( + 'FE0 fence baseline: the provisioned deployment is open and minting work', + async () => { + const initial = await readFence(); + assert( + initial.state === 'open' && + initial.proofKey === undefined && + initial.proofRunId === undefined, + 'provisioning seeded an explicit open fence with no proof binding', + initial, + ); + // Two suspended runs minted while OPEN. One is drained through the + // fence's draining state, the other is left for the lock to refuse — + // both have to exist before the first transition, because a fenced + // deployment mints nothing. + const draining = await startFencedRun(); + const locked = await startFencedRun(); + assert( + draining.runId !== locked.runId, + 'fence probes need two distinct runs', + { draining, locked }, + ); + return { draining, locked }; + }, + ); + + await step( + 'FE1 draining: new starts are refused EXECUTION_FENCED while an existing ' + + "run's approval still resumes it to completion", + async () => { + await moveFence('open', 'draining'); + + const refusedStart = await http('POST', '/runs', { + body: RUN_BODY, + headers: AUTH.operator, + }); + assertFenced('a run start under draining', refusedStart, 'draining'); + + // The whole point of draining: outstanding work must still be able to + // finish, or the drain can never complete. A different actor decides + // (separation of duties), and the resume runs the workflow to success. + const decided = await http( + 'POST', + `/api/approvals/${fenceRuns.draining.approvalId}/decide`, + { headers: AUTH.reviewer, body: { decision: 'approve' } }, + ); + assert( + decided.status === 200 && + decided.body.resume?.summary?.status === 'success', + 'a draining deployment must still resume an already-suspended run', + decided, + ); + }, + ); + + await step( + 'FE2 migration-locked: resume, approval decide, and start are all refused, ' + + 'and the refused decision is not committed', + async () => { + await moveFence('draining', 'migration-locked'); + + const refusedStart = await http('POST', '/runs', { + body: RUN_BODY, + headers: AUTH.operator, + }); + assertFenced( + 'a run start under migration-locked', + refusedStart, + 'migration-locked', + ); + + const refusedDecide = await http( + 'POST', + `/api/approvals/${fenceRuns.locked.approvalId}/decide`, + { headers: AUTH.reviewer, body: { decision: 'approve' } }, + ); + assertFenced( + 'an approval decision under migration-locked', + refusedDecide, + 'migration-locked', + ); + + // The gate sits BEFORE the decision's compare-and-set: a decision that + // committed here would be durably recorded on a deployment that can never + // act on it, and the deployment taking over would inherit a decided + // approval with no resume behind it. + const stillPending = await http( + 'GET', + `/api/approvals/${fenceRuns.locked.approvalId}`, + { headers: AUTH.viewer }, + ); + assert( + stillPending.status === 200 && + stillPending.body.status === 'pending' && + stillPending.body.decidedBy === undefined, + 'the refused decision left the approval untouched', + stillPending, + ); + + // The run DO's own resume gate, reached directly rather than through the + // approval service — the fence has to hold on both paths. + const refusedResume = await http( + 'POST', + `/runs/${RUN_BODY.workflowId}/${encodeURIComponent(fenceRuns.locked.runId)}/resume`, + { + headers: AUTH.operator, + body: { + step: fenceRuns.locked.step, + resumeData: { approved: true }, + }, + }, + ); + assertFenced( + 'a raw resume under migration-locked', + refusedResume, + 'migration-locked', + ); + }, + ); + + await step( + 'FE3 fence persistence: the lock survives a workerd kill+restart and the ' + + 'restarted process still refuses to mint work', + async () => { + await killServer(currentServer); + await launchServer( + 'fence-restart', + stateDir, + join(tmpDir, 'fence-restart.log'), + ); + assert( + !/address already in use/i.test(currentServer.chunks.join('')), + 'fence-restart log must not contain "address already in use" (orphan trap)', + ); + + // Nothing in the new process has ever seen a fence transition: the only + // thing that carried the lock across process death is the D1 row. + const restored = await readFence(); + assert( + restored.state === 'migration-locked', + 'the fence state survived process death', + restored, + ); + const refusedStart = await http('POST', '/runs', { + body: RUN_BODY, + headers: AUTH.operator, + }); + assertFenced( + 'a run start on the restarted locked deployment', + refusedStart, + 'migration-locked', + ); + }, + ); + + await step( + 'FE4 reopen: migration-locked -> open restores minting, and the approval ' + + 'the lock refused now completes its run', + async () => { + await moveFence('migration-locked', 'open'); + assert( + (await readFence()).state === 'open', + 'the reopened fence reads back as open', + ); + + const started = await startFencedRun(); + assert( + started.runId !== fenceRuns.locked.runId, + 'the reopened deployment minted a fresh run', + started, + ); + + // Nothing was lost under the lock: the approval that was refused is still + // pending and still resumes its run. + const decided = await http( + 'POST', + `/api/approvals/${fenceRuns.locked.approvalId}/decide`, + { headers: AUTH.reviewer, body: { decision: 'approve' } }, + ); + assert( + decided.status === 200 && + decided.body.resume?.summary?.status === 'success', + 'the approval refused under the lock completes once the fence reopens', + decided, + ); + }, + ); + + // --- Idempotent start (F3): exactly-once, on real workerd ----------------- + // Unit tests can prove the reservation's compare-and-set over node:sqlite. + // What they cannot prove is that a reservation written by one PROCESS still + // converges a retry in the next one, that two genuinely concurrent requests + // reach one run through real D1 rather than a synchronous test double, and + // that the fence's proof-only state admits exactly the start carrying its + // nominated key once the whole router chain is in the way. + + const idempotentRun = await step( + 'FI1 idempotent start: a retry after a workerd kill+restart returns the ' + + 'SAME run, and the paid first step ran exactly ONCE', + async () => { + const first = await startCountedWithKey('spike-key-1', 'fi1'); + assert( + first.status === 200 && first.body.status === 'suspended', + 'the first keyed start must run normally', + first, + ); + // The start responded with the SUSPENDED summary, so the counting step + // has already run and its row is durable. Anything other than 1 here + // would mean the probe itself is not measuring what it claims to. + assert( + (await executionCount('fi1')) === 1, + 'the first keyed start must execute the counting step exactly once', + ); + + // Process death between the response and the retry — the case a client + // cannot tell from a lost response. + await killServer(currentServer); + await launchServer( + 'idempotent-restart', + stateDir, + join(tmpDir, 'idempotent-restart.log'), + ); + + const retry = await startCountedWithKey('spike-key-1', 'fi1'); + assert( + retry.status === 200 && retry.body.runId === first.body.runId, + 'the retry must replay the first run, not start a second', + { first: first.body, retry: retry.body }, + ); + // The claim this probe exists for, stated as EXECUTIONS rather than as + // run ids: the retry answered from the reservation and ran nothing. + // Nothing in the restarted process had ever seen this key — the only + // thing that carried it across process death is the D1 row. + const executions = await executionCount('fi1'); + assert( + executions === 1, + 'a retry after process death must execute the first step no second time', + { executions, first: first.body, retry: retry.body }, + ); + return { runId: first.body.runId, approvalId: first.body.approval?.id }; + }, + ); + + await step( + 'FI2 concurrent same-key starts: two in-flight requests produce ONE run ' + + 'and ONE execution', + async () => { + // The cross-isolate race a kill-and-retry harness cannot fake: neither + // request has seen the other, and only the reservation's CAS is between + // them. On the agent surface this is the ONLY thing between them, because + // two same-key starts naming different threads are two different Durable + // Objects with no shared lock at all. + const [a, b] = await Promise.all([ + startCountedWithKey('spike-key-2', 'fi2'), + startCountedWithKey('spike-key-2', 'fi2'), + ]); + const responses = [a, b]; + const accepted = responses.filter((response) => response.status === 200); + const runIds = new Set(accepted.map((response) => response.body.runId)); + // At least one caller must be ANSWERED, not merely refused consistently: + // a burst in which both requests were told to retry would satisfy every + // "no second run" assertion below while proving nothing about the start. + assert( + accepted.length >= 1, + 'at least one concurrent same-key start must be answered with a run', + { + a: { status: a.status, body: a.body }, + b: { status: b.status, body: b.body }, + }, + ); + assert( + runIds.size === 1, + 'two concurrent same-key starts must resolve to exactly one run', + { + a: { status: a.status, body: a.body }, + b: { status: b.status, body: b.body }, + }, + ); + // The loser is allowed to replay (200), to be told the winner is still + // working (503 IDEMPOTENT_START_PENDING), or — inside the window between + // the winning claim and its dispatch — to be told UNRESOLVABLE (409). + // What it may never be is a second run, and what it may never carry is a + // code outside the published taxonomy: a collapsed or generic refusal + // would read to a client as "retry", which is exactly the advice that + // charges twice. + const refusalCodes = new Map(); + for (const response of responses) { + if (response.status === 200) continue; + const code = response.body?.reason?.code; + assert( + code === 'IDEMPOTENT_START_PENDING' || + code === 'IDEMPOTENT_START_UNRESOLVABLE', + 'a concurrent same-key start was refused outside the taxonomy', + { status: response.status, body: response.body }, + ); + refusalCodes.set(code, (refusalCodes.get(code) ?? 0) + 1); + } + // One 200 carries the suspended summary, so the counting step has + // already run by the time both requests have settled — no polling, and + // no window in which a second execution could still be in flight. + const executions = await executionCount('fi2'); + assert( + executions === 1, + 'a concurrent same-key burst must execute the first step exactly once', + { + executions, + accepted: accepted.length, + refusals: Object.fromEntries(refusalCodes), + }, + ); + console.log( + ` answered: ${accepted.length}, refusals: ${JSON.stringify(Object.fromEntries(refusalCodes))}`, + ); + }, + ); + + await step( + 'FI3 proof-only: the nominated key is admitted, every other start is ' + + 'refused, and the proof binds to exactly one run', + async () => { + await moveFence('open', 'proof-only', 'spike-proof-key'); + const nominated = await readFence(); + assert( + nominated.state === 'proof-only' && + nominated.proofKey === 'spike-proof-key' && + nominated.proofRunId === undefined, + 'entering proof-only nominates a key and binds no run yet', + nominated, + ); + + // A start with no key at all: refused, exactly as under the lock. + const unkeyed = await http('POST', '/runs', { + body: RUN_BODY, + headers: AUTH.operator, + }); + assertFenced('an unkeyed start under proof-only', unkeyed, 'proof-only'); + + // A start carrying a DIFFERENT key: also refused. The key is not a + // password, it is a nomination — only the operator's own key matches. + const wrongKey = await startWithKey('spike-wrong-key'); + assertFenced( + 'a start carrying the wrong key under proof-only', + wrongKey, + 'proof-only', + ); + + // The nominated start runs, and binds the proof. + const proof = await startWithKey('spike-proof-key'); + assert( + proof.status === 200 && proof.body.status === 'suspended', + 'the nominated proof start must be admitted', + proof, + ); + const bound = await readFence(); + assert( + bound.proofRunId === proof.body.runId, + 'the admitted proof start binds the fence to its run', + { bound, proof: proof.body }, + ); + + // Reopening restores ordinary minting, and clears the proof binding so + // the next proof cannot inherit this one's run. + await moveFence('proof-only', 'open'); + const reopened = await readFence(); + assert( + reopened.state === 'open' && + reopened.proofKey === undefined && + reopened.proofRunId === undefined, + 'reopening clears the proof nomination and its binding', + reopened, + ); + const afterReopen = await startWithKey('spike-key-3'); + assert( + afterReopen.status === 200 && + afterReopen.body.runId !== proof.body.runId && + afterReopen.body.runId !== idempotentRun.runId, + 'the reopened deployment mints fresh runs again', + afterReopen, + ); + }, + ); + + // --- Drain inventory (F2): the migration proof, on real workerd ----------- + // Unit tests can prove the queries are pure SELECTs against the real schemas. + // What they cannot prove is that those queries read the SAME D1 that real + // HTTP traffic on real workerd wrote its runs, approvals, reservations, + // notifications, and tasks into — and that the numbers MOVE as that work + // drains. An inventory that reported the right shape over the wrong database + // would pass every unit test and certify a deployment that still owed work. + + await step( + 'FV1 inventory index: every category, its class, the states no query can ' + + 'see, and the rule an empty answer means something under', + async () => { + const index = await readInventoryIndex(); + const work = index.categories + .filter((entry) => entry.class === 'work') + .map((entry) => entry.category); + const standing = index.categories + .filter((entry) => entry.class === 'standing') + .map((entry) => entry.category); + assert( + work.length === 7 && standing.length === 2, + 'the index splits work from standing configuration', + { work, standing }, + ); + assert( + index.unenumerable.some( + (entry) => entry.name === 'run-owner-recovery-journal', + ), + 'the Durable Object journal window is DECLARED, not silently omitted', + index.unenumerable, + ); + assert( + Array.isArray(index.drainProof?.reachableFrom) && + index.drainProof.reachableFrom.length === 1 && + index.drainProof.reachableFrom[0] === 'draining', + "an empty work set only means something from 'draining'", + index.drainProof, + ); + }, + ); + + await step( + 'FV2 drain proof: seeded work appears while draining, leaves as it is ' + + 'finished, and reads empty across TWO consecutive sweeps before the lock', + async () => { + const index = await readInventoryIndex(); + + // A BASELINE, because this deployment has been running scenarios for the + // whole spike and legitimately still holds work from them. The proof + // below is over the seeded DELTA: these exact rows appear, and these + // exact rows are gone — which is the same claim an empty sweep makes, + // stated about rows whose lifecycle this step controls. + const before = await sweepInventoryWork(index); + + // Seed: a suspended run with a pending approval, and a keyed start + // (a second suspended run PLUS a start reservation). + const drained = await startFencedRun(); + const keyed = await startCountedWithKey('spike-inventory-key', 'inv'); + assert( + keyed.status === 200 && keyed.body.status === 'suspended', + 'the keyed seed run suspended', + keyed.body, + ); + + // Draining: the operator has stopped new work and is now proving what is + // left. Reads stay open in every state, which is the whole point. + await moveFence('open', 'draining'); + const seeded = await sweepInventoryWork(index); + + assert( + seeded.runs.keys.some((key) => key.endsWith(`/${drained.runId}`)) && + seeded.runs.keys.some((key) => key.endsWith(`/${keyed.body.runId}`)), + 'both seeded runs are reported as outstanding work', + seeded.runs, + ); + assert( + seeded['approvals-waiting'].keys.includes(drained.approvalId), + 'the pending approval is reported as outstanding work', + seeded['approvals-waiting'], + ); + assert( + seeded['start-reservations'].keys.includes('spike-inventory-key'), + 'the unsettled start reservation is reported as outstanding work', + seeded['start-reservations'], + ); + assert( + seeded.runs.count >= before.runs.count + 2, + 'the run count moved by exactly the work that was seeded', + { before: before.runs.count, seeded: seeded.runs.count }, + ); + + // The categories the earlier scenarios populated are READ here too, over + // the same real D1: their sub-counts are what tell an operator a parked + // task from one awaiting a webhook, and a due notification from one + // scheduled for later. + assert( + typeof seeded['background-tasks'].count === 'number' && + typeof seeded['background-tasks'].totals?.fenceSuspended === 'number', + 'background tasks report a fence-parked sub-count', + seeded['background-tasks'], + ); + assert( + typeof seeded['pending-notifications'].count === 'number' && + typeof seeded['pending-notifications'].totals?.notDue === 'number', + 'pending notifications separate what is due from what is not', + seeded['pending-notifications'], + ); + + // Drain: a draining deployment still finishes what it already has. Both + // seeded runs complete, and the reservation settles with its run. + const decided = await http( + 'POST', + `/api/approvals/${drained.approvalId}/decide`, + { headers: AUTH.reviewer, body: { decision: 'approve' } }, + ); + assert( + decided.status === 200 && + decided.body.resume?.summary?.status === 'success', + 'the draining deployment resumed the seeded run to success', + decided, + ); + const keyedApprovalId = keyed.body.approval?.id; + assert( + typeof keyedApprovalId === 'string', + 'the keyed seed run filed an approval to drain', + keyed.body, + ); + const keyedDecided = await http( + 'POST', + `/api/approvals/${keyedApprovalId}/decide`, + { headers: AUTH.reviewer, body: { decision: 'approve' } }, + ); + assert( + keyedDecided.status === 200 && + keyedDecided.body.resume?.summary?.status === 'success', + 'the keyed seed run also resumed to success', + keyedDecided, + ); + + // The proof: the seeded rows are gone, and STILL gone on a second full + // sweep. One sweep is a point-in-time reading: traffic can still create + // or complete work while it runs, so the contract asks for two. + const gone = (sweep) => + !sweep.runs.keys.some((key) => key.endsWith(`/${drained.runId}`)) && + !sweep.runs.keys.some((key) => key.endsWith(`/${keyed.body.runId}`)) && + !sweep['approvals-waiting'].keys.includes(drained.approvalId) && + !sweep['approvals-waiting'].keys.includes(keyedApprovalId) && + !sweep['start-reservations'].keys.includes('spike-inventory-key'); + const first = await sweepInventoryWork(index); + assert(gone(first), 'the first sweep no longer reports the seeded work', { + runs: first.runs.keys, + approvals: first['approvals-waiting'].keys, + reservations: first['start-reservations'].keys, + }); + const second = await sweepInventoryWork(index); + assert( + gone(second), + 'the second consecutive sweep agrees — the drain proof holds', + { + runs: second.runs.keys, + approvals: second['approvals-waiting'].keys, + reservations: second['start-reservations'].keys, + }, + ); + + // Standing configuration is NOT drained: a schedule and a provider + // subscription survive the migration, and demanding they empty would make + // the proof unreachable rather than strict. + const schedules = await sweepInventoryCategory('schedules'); + const subscriptions = await sweepInventoryCategory( + 'signal-subscriptions', + ); + assert( + schedules.class === 'standing' && schedules.keys.length >= 1, + 'schedules are still reported while the drain proof passes', + schedules, + ); + assert( + subscriptions.class === 'standing' && subscriptions.keys.length >= 1, + 'provider subscriptions are still reported while the proof passes', + subscriptions, + ); + + // Lock: the proof held, so the deployment may stop executing. The + // inventory keeps answering — reads are ungated in every state, which is + // what lets an operator verify the lock they just took. + await moveFence('draining', 'migration-locked'); + const locked = await sweepInventoryWork(index); + assert( + gone(locked), + 'the inventory still answers under the lock, with the same verdict', + locked.runs.keys, + ); + const refusedStart = await http('POST', '/runs', { + body: RUN_BODY, + headers: AUTH.operator, + }); + assertFenced( + 'a run start after the drain proof locked the deployment', + refusedStart, + 'migration-locked', + ); + await moveFence('migration-locked', 'open'); + }, + ); + await step( 'S deployment sentinel mismatch: a freshly started Worker refuses a D1 ' + 'provisioned for another deployment', @@ -2381,7 +3144,31 @@ try { "reserved deadline key armed the run DO's own fenced wake, and after a " + 'kill+restart that wake fired on the restarted object and resumed the run ' + 'ITSELF with the timeout envelope under the system principal — no client ' + - 'called resume. Finally, a fresh Worker refused a D1 sentinel stamped for ' + + 'called resume. The deployment execution fence (F1): provisioning seeded ' + + 'an explicit open fence, draining refused new starts with 503 ' + + 'EXECUTION_FENCED while still resuming an outstanding approval to ' + + 'success, migration-locked refused starts, raw resumes AND approval ' + + 'decisions (leaving the refused decision uncommitted), that lock survived ' + + 'a workerd kill+restart and still refused to mint, and reopening it ' + + 'restored minting and completed the approval the lock had refused. ' + + 'Owner-bound idempotent start (F3): a keyed start retried after a ' + + 'workerd kill+restart replayed the SAME run rather than starting a ' + + 'second, two genuinely concurrent same-key starts resolved to ONE ' + + 'run — both proved by a durable D1 execution counter reading exactly 1, ' + + 'with every concurrent refusal inside the published taxonomy — ' + + 'and under proof-only exactly the start carrying the nominated ' + + 'key was admitted and bound the fence to its run while unkeyed and ' + + 'wrong-keyed starts were refused. ' + + 'The drain inventory (F2): the index declared every category with its ' + + 'class, the Durable Object journal window it cannot see, and the rule an ' + + 'empty answer means something under; seeded runs, an approval, and a ' + + 'start reservation appeared as outstanding work while draining, left as ' + + 'each was finished, and read absent across TWO consecutive full sweeps ' + + 'paged through the keyset — after which the deployment locked, still ' + + 'answered the same inventory, and refused to mint, while its schedules ' + + 'and provider subscriptions kept being reported as standing ' + + 'configuration a migration carries rather than drains. ' + + 'Finally, a fresh Worker refused a D1 sentinel stamped for ' + 'another deployment with 503 before authentication or routing.', ); } catch (error) { diff --git a/packages/flowsafe/spike/durability-benchmark.worker.ts b/packages/flowsafe/spike/durability-benchmark.worker.ts index f19b1fb..f14b1d0 100644 --- a/packages/flowsafe/spike/durability-benchmark.worker.ts +++ b/packages/flowsafe/spike/durability-benchmark.worker.ts @@ -24,6 +24,7 @@ import { import { DurableObjectRunner, ensureDeploymentIdentityBindings, + executionFenceFor, init, isPathSafeId, type RunnerRuntime, @@ -285,6 +286,10 @@ export class BenchmarkFlowsafeRunner extends DurableObjectRunner { protected runLifecycle(env: Env) { const service = new ApprovalService({ store: approvalFactory(env.DB).store(), + // Deliberately unfenced: this service only abandons approvals for a + // terminated run, which is allowed in every fence state because it + // removes future work rather than starting any. It never decides. + executionFence: 'none', }); return { abandonApprovals: ( @@ -415,6 +420,9 @@ function flowsafeService(env: Env): ApprovalService { return new ApprovalService({ store: approvalFactory(env.DB).store(), resumeRun: topology.resumeRecord, + // This one DECIDES, and decide() commits before it resumes: same database, + // same fence as the runs it moves. + executionFence: executionFenceFor(env.DB as unknown as never), }); } diff --git a/packages/flowsafe/spike/worker.ts b/packages/flowsafe/spike/worker.ts index 9e641d7..5c66414 100644 --- a/packages/flowsafe/spike/worker.ts +++ b/packages/flowsafe/spike/worker.ts @@ -120,22 +120,30 @@ import { createBackgroundTaskD1Domains, } from '../src/background-tasks/index.js'; import { + assertExecutionFenceState, createD1Storage, createHostPubSub, + DeploymentInventory, DurableObjectRunner, doErrorResponse, + type ExecutionFenceStore, ensureDeploymentIdentityBindings, + executionFenceFor, + executionFenceReadingPayload, HubDurableObject, type InitResult, init, + isInventoryCategory, isPathSafeId, isSuspensionTimeoutResumeData, mintThreadId, type RunnerRuntime, resourceIdFromKey, + type StartIdempotencyStore, SUSPENSION_DEADLINE_PAYLOAD_KEY, SUSPENSION_TIMEOUT_RESUME_KEY, stampDeploymentIdentityRequest, + startIdempotencyFor, ThreadDurableObject, type ThreadScope, verifyDurableObjectDeploymentIdentity, @@ -220,6 +228,33 @@ interface Env { STREAM_TICKET_SECRET: string; } +/** + * ONE execution-fence store per D1 binding, for every surface in this worker. + * + * The spike composes its routers, ticks, and Durable Objects by hand rather + * than through createFlowsafeWorker, so nothing else would keep the admin probe + * that MOVES the fence, the schedule ticks that read it before claiming, and + * the runtimes that obey it pointed at the same database. The memo itself is + * the package's (`executionFenceFor`), keyed on the binding for the same reason + * host-kit's composer is: the fence belongs to the database, not to the request + * that reached it. This wrapper only unwraps `env` — which is what every call + * site here holds — and carries the workers-types-to-structural cast the rest + * of this worker makes on the same binding. + */ +function executionFenceForEnv(env: Env): ExecutionFenceStore { + return executionFenceFor(env.DB as unknown as never); +} + +/** + * The same one-store-per-binding rule for start reservations: the run router + * and every agent topology below reserve here, and the runtimes inside the run + * and thread objects settle there, and both must be the store built from THIS + * binding. + */ +function startIdempotencyForEnv(env: Env): StartIdempotencyStore { + return startIdempotencyFor(env.DB as unknown as never); +} + function fetchDeploymentObject( stub: ReturnType, env: Env, @@ -593,6 +628,19 @@ const SPIKE_ACTORS = new Map([ ['other-viewer', { id: 'vera', role: 'viewer' }], ]); +// --- Idempotent-start execution counter (FI1/FI2) --------------------------- +// The reservation's claim is not "one run id came back twice" — it is that the +// paid first step of a keyed start EXECUTED ONCE. A repeated run id is only +// evidence of that if nothing else could have run, which is an inference. This +// workflow's first step writes a durable D1 row instead, so the spike can count +// executions directly across a process death and across a concurrent burst. +const COUNTED_WORKFLOW_ID = 'demo-idempotent'; +const EXECUTION_COUNT_TABLE = 'spike_execution_count'; +const EXECUTION_COUNT_DDL = `CREATE TABLE IF NOT EXISTS ${EXECUTION_COUNT_TABLE} ( + id TEXT PRIMARY KEY, + executions INTEGER NOT NULL + )`; + const WORKFLOWS: ReadonlyArray = [ { id: 'demo-approval', @@ -615,6 +663,13 @@ const WORKFLOWS: ReadonlyArray = [ 'one step that suspends with a per-suspension deadline, so the run resumes itself when the awaited signal never arrives', sampleInput: { topic: 'launch', deadlineMs: 10_000 }, }, + { + id: COUNTED_WORKFLOW_ID, + title: 'Demo idempotent start', + description: + 'demo-approval with a counting first step, so an idempotent start can be proved by EXECUTIONS rather than by run ids', + sampleInput: { topic: 'launch', counterId: 'probe' }, + }, ]; const scheduleTargetPolicy = createScheduleTargetPolicy({ workflows: [...WORKFLOWS, { id: 'sched-echo' }], @@ -745,6 +800,46 @@ function defineWorkflows(env: Env): RunnerRuntime { .then(publish) .commit(); + // FI1/FI2: demo-approval's shape with a COUNTING first step. The step is + // ordinary paid work as far as the runtime is concerned — it writes to D1 + // before the run suspends — so a second execution of this key would show up + // as a second row increment whatever the run ids said. + const countedResearch = createStep({ + id: 'counted-research', + inputSchema: z.object({ topic: z.string(), counterId: z.string() }), + outputSchema: z.object({ topic: z.string(), notes: z.string() }), + execute: async ({ inputData }) => { + // Lazy DDL rather than a provisioning step: this table belongs to the + // probe, not to the deployment, and creating it here keeps its one + // definition beside its one writer. + await env.DB.prepare(EXECUTION_COUNT_DDL).run(); + await env.DB.prepare( + `INSERT INTO ${EXECUTION_COUNT_TABLE} (id, executions) VALUES (?, 1) + ON CONFLICT(id) DO UPDATE SET executions = executions + 1`, + ) + .bind(inputData.counterId) + .run(); + return { + topic: inputData.topic, + notes: `research notes for ${inputData.topic}`, + }; + }, + }); + + createWorkflow({ + id: COUNTED_WORKFLOW_ID, + inputSchema: z.object({ topic: z.string(), counterId: z.string() }), + outputSchema: z.object({ + topic: z.string(), + published: z.boolean(), + approvedBy: z.string().optional(), + }), + }) + .then(countedResearch) + .then(approval) + .then(publish) + .commit(); + // Track A agent-gate probe: the SAME grant-only loop, but the gate suspends // with the durable-agent tool-call shape (R-003) — { type:'approval', // toolName, ... } with NO explicit `connectors` array. host-kit's @@ -993,6 +1088,12 @@ export class DemoRunner extends DurableObjectRunner { protected runLifecycle(env: Env) { const service = new ApprovalService({ store: approvalStoreFactory(env.DB).store(), + // Deliberately unfenced: the only thing this service is used for is + // abandonApprovalsForRun below, and abandoning is a terminate-path + // operation that stays allowed in every fence state — it removes future + // work, which is the direction a drain is going. It never decides, so it + // never commits a decision a locked deployment could not resume. + executionFence: 'none', }); return { abandonApprovals: ( @@ -1054,6 +1155,7 @@ export class DemoBackgroundTasks { mastra, pubsub, execution: true, + executionFence: executionFenceForEnv(this.#env), executors: { bgProbe: { execute: async (args) => { @@ -1124,7 +1226,7 @@ export class DemoBackgroundTasks { if (!isPathSafeId(body.runId)) { return json({ error: 'path-safe parent runId required' }, 404); } - const queued = await host.manager.enqueue({ + const queued = await host.enqueue({ runId: body.runId, toolName: 'bgProbe', toolCallId: `call-${crypto.randomUUID()}`, @@ -1138,7 +1240,7 @@ export class DemoBackgroundTasks { return json({ taskId: queued.task.id, status: queued.task.status }); } if (request.method === 'GET' && url.pathname.startsWith('/task/')) { - const task = await host.manager.getTask( + const task = await host.getTask( decodeURIComponent(url.pathname.slice('/task/'.length)), ); return task ? json(task) : json({ error: 'not found' }, 404); @@ -1207,6 +1309,10 @@ export class DemoThread extends ThreadDurableObject { approvalService: () => { this.#approvalService ??= new ApprovalService({ store: approvals, + // The thread's own agent-approval service decides, and decide() + // COMMITS before it resumes, so it gates on the same store as the + // thread runtime beside it. + executionFence: executionFenceForEnv(env), stream: (event) => createHubTopology( this.env.HUB, @@ -1226,6 +1332,15 @@ export class DemoThread extends ThreadDurableObject { requestContextForRun: agentHost.requestContextForRun( approvalGrantProvider(approvals), ), + // The composite store hides the binding init would have fenced from, + // so this thread DO names it: the fence must live in the SAME database + // as the state it fences. + executionFence: executionFenceForEnv(env), + // Same reasoning, and the same binding: the agent topology reserves + // keyed starts into THIS store, so the runtime that sees an agent run + // reach terminal has to be the one that can mark them spent. Wiring + // 'none' here would reserve and claim normally and then never settle. + startIdempotency: startIdempotencyForEnv(env), }, ); this.#threadInit = threadInit; @@ -1510,6 +1625,7 @@ export class DemoSignalProviderHost extends SignalProviderHost { env.DEPLOYMENT_IDENTITY_SECRET, ), providers: spikeProviders(), + executionFence: executionFenceForEnv(env), }; } } @@ -1538,6 +1654,7 @@ function webhookRouter(env: Env): ReturnType { audit: (event) => { sigpAudit.push(event); }, + executionFence: executionFenceForEnv(env), }); webhookRouters.set(env.DB, router); } @@ -1715,7 +1832,14 @@ function actorContextForPrincipal( principal, storeFactory: factory, deploymentTag: env.DEPLOYMENT_TENANT, - buildService: (store) => new ApprovalService({ store }), + buildService: (store) => + new ApprovalService({ + store, + // The twin of buildApprovalService below, and fenced for the same + // reason: these contexts decide approvals, and decide() COMMITS before + // it resumes. Same database, same store. + executionFence: executionFenceForEnv(env), + }), }); } @@ -1779,6 +1903,10 @@ function buildApprovalService( topology: createAgentThreadTopology( env.THREAD, env.DEPLOYMENT_IDENTITY_SECRET, + { + startIdempotency: startIdempotencyForEnv(env), + executionFence: executionFenceForEnv(env), + }, ), contextForPrincipal: (principal, record) => { const target = record.resumeTarget; @@ -1800,6 +1928,11 @@ function buildApprovalService( // caller keeps the publish alive with ctx.waitUntil (see fetch below). stream, resumeRun, + // DECIDE commits and then resumes, so the fence has to be consulted at the + // service — before the CAS — not left to the run DO's own resume gate. A + // decision that committed against a locked deployment would be durable with + // nothing behind it. Same database as the runs it gates. + executionFence: executionFenceForEnv(env), }); } @@ -1926,6 +2059,7 @@ async function handleBackgroundTaskProbe( mastra: bgMastra(env), pubsub: createHostPubSub(), executors: {}, + executionFence: executionFenceForEnv(env), }); await host.boot(); return json({ recovered: true }); @@ -2078,6 +2212,7 @@ async function handleGoalProbe( audit: (event) => { events.push(event); }, + executionFence: executionFenceForEnv(env), ...(maxRunsCap !== undefined ? { maxRunsCap } : {}), }); const res = await router( @@ -2209,6 +2344,7 @@ async function handleScheduleProbe( store, targetPolicy: scheduleTargetPolicy, deploymentTag: env.DEPLOYMENT_TENANT, + executionFence: executionFenceForEnv(env), start: async ({ scheduleId, dispatchId, @@ -2300,6 +2436,7 @@ async function handleScheduleProbe( store, targetPolicy: scheduleTargetPolicy, deploymentTag: env.DEPLOYMENT_TENANT, + executionFence: executionFenceForEnv(env), // Fire through the DO topology (the production path) — the tick mints the // opaque runId and the DO runs sched-echo, echoing its leg context keys. start: async ({ @@ -2368,6 +2505,10 @@ async function handleScheduleProbe( const topology = createAgentThreadTopology( env.THREAD, env.DEPLOYMENT_IDENTITY_SECRET, + { + startIdempotency: startIdempotencyForEnv(env), + executionFence: executionFenceForEnv(env), + }, ); await ownerContext.claimResource('thread', threadId); await ownerContext.claimResource('resource', resourceId); @@ -2477,6 +2618,7 @@ async function handleScheduleProbe( store, targetPolicy: scheduleTargetPolicy, deploymentTag: env.DEPLOYMENT_TENANT, + executionFence: executionFenceForEnv(env), // Tripwire, the same shape as the scheduleId guards below: this probe // creates only an agent-target schedule, so a workflow fire means the // entry-level isolation broke. It is a tripwire, not the guarantee — the @@ -2649,6 +2791,120 @@ async function handleSuspensionDeadlineProbe( return json({ status: response.status, armed: await response.json() }); } +// --- Idempotent-start execution count probe (FI1/FI2) ---------------------- +// Reads the counter `counted-research` writes. LOCAL-ONLY and unauthenticated, +// like every other spike probe: it exposes nothing a run's own status does not, +// and exists because no published surface reports how many times a step ran. +async function handleExecutionCountProbe( + request: Request, + env: Env, +): Promise { + const url = new URL(request.url); + if (request.method !== 'GET' || url.pathname !== '/idempotent/executions') { + return null; + } + const counterId = url.searchParams.get('counterId'); + if (!isPathSafeId(counterId)) { + return json({ error: 'a path-safe counterId is required' }, 404); + } + try { + const { results } = await env.DB.prepare( + `SELECT executions FROM ${EXECUTION_COUNT_TABLE} WHERE id = ?`, + ) + .bind(counterId) + .all<{ executions: number }>(); + return json({ executions: results[0]?.executions ?? 0 }); + } catch (error) { + // The table is created by the first counted step, so before any counted + // run has executed, ZERO executions is the truth rather than a fault. + if (!/no such table/i.test(String(error))) throw error; + return json({ executions: 0 }); + } +} + +// --- Execution fence control probe (F1) ------------------------------------ +// LOCAL-ONLY worker-level fence control channel, in the same shape the +// published admin route serves (`GET`/`POST /admin/execution-fence`, CAS on +// `expected`). +// +// It is spike-local rather than the host-kit route because this worker composes +// its routers by hand and never calls createFlowsafeWorker, so it configures no +// MAINTENANCE_ADMIN_SECRET and mounts no /admin surface at all. What the spike +// exists to prove is the part unit tests cannot: that the fence state is +// DURABLE across process death and that the enforcement points refuse real HTTP +// requests on real workerd. Both go through the same ExecutionFenceStore the +// published route drives; only the authentication in front of it differs, and +// that is covered by flowsafe-worker.test.ts. +async function handleExecutionFenceProbe( + request: Request, + env: Env, +): Promise { + const url = new URL(request.url); + if (url.pathname !== '/admin/execution-fence') return null; + const fence = executionFenceForEnv(env); + try { + if (request.method === 'GET') { + return json(executionFenceReadingPayload(await fence.read())); + } + if (request.method !== 'POST') { + return json({ error: 'method not allowed' }, 405); + } + const body = (await request.json()) as { + expected?: unknown; + next?: unknown; + proofKey?: unknown; + }; + const reading = await fence.transition({ + expected: assertExecutionFenceState(body.expected, 'expected'), + next: assertExecutionFenceState(body.next, 'next'), + ...(body.proofKey === undefined ? {} : { proofKey: body.proofKey }), + }); + return json({ state: reading.state }); + } catch (error) { + return doErrorResponse(error); + } +} + +// --- Drain inventory probe (F2) -------------------------------------------- +// LOCAL-ONLY worker-level read of the deployment's outstanding work, in the +// same shape the published `GET /admin/inventory` route serves (index with no +// category, keyset page with one). Spike-local for the same reason the fence +// probe is: this worker composes its routers by hand and mounts no +// authenticated /admin surface. +// +// What the spike proves that unit tests cannot: that the inventory reads the +// SAME D1 the runs, notifications, tasks, and reservations were really written +// to by real workerd traffic, that its counts move as that work drains, and +// that it stays answerable while the fence is refusing everything else. +async function handleInventoryProbe( + request: Request, + env: Env, +): Promise { + const url = new URL(request.url); + if (url.pathname !== '/admin/inventory') return null; + if (request.method !== 'GET') { + return json({ error: 'method not allowed' }, 405); + } + const inventory = new DeploymentInventory(env.DB as unknown as never); + try { + const category = url.searchParams.get('category'); + if (category === null || category === '') return json(inventory.index()); + if (!isInventoryCategory(category)) { + return json({ error: `unknown inventory category '${category}'` }, 400); + } + const cursor = url.searchParams.get('cursor'); + const limit = url.searchParams.get('limit'); + return json( + await inventory.read(category, { + ...(cursor === null ? {} : { cursor }), + ...(limit === null ? {} : { limit: Number(limit) }), + }), + ); + } catch (error) { + return doErrorResponse(error); + } +} + const handler: ExportedHandler = { async fetch( request: CfRequest, @@ -2690,6 +2946,17 @@ const handler: ExportedHandler = { env.DEPLOYMENT_IDENTITY_SECRET, ); + // The fence control channel, ahead of every router: an operator has to be + // able to move the fence on a deployment the fence is already refusing. + const fenceProbe = await handleExecutionFenceProbe(routed, env); + if (fenceProbe) return fenceProbe; + + // The drain proof, beside the control that holds the drain open: an + // operator has to be able to READ what remains on a deployment that is + // refusing everything else. + const inventoryProbe = await handleInventoryProbe(routed, env); + if (inventoryProbe) return inventoryProbe; + const agentProbeResponse = await handleLiveAgentRoute(routed, env, resolve); if (agentProbeResponse) return agentProbeResponse; @@ -2723,6 +2990,11 @@ const handler: ExportedHandler = { const deadlineProbe = await handleSuspensionDeadlineProbe(routed, env); if (deadlineProbe) return deadlineProbe; + // Idempotent-start execution count (FI1/FI2) — local, unauthenticated, + // ahead of the routers. + const executionCountProbe = await handleExecutionCountProbe(routed, env); + if (executionCountProbe) return executionCountProbe; + // Track E webhook ingress: the github webhook route TERMINATES on the Worker, // signature-authed (not bearer), route-absent when its secret is unset. const webhookResponse = await webhookRouter(env)(routed); @@ -2748,6 +3020,10 @@ const handler: ExportedHandler = { topology: createAgentThreadTopology( env.THREAD, env.DEPLOYMENT_IDENTITY_SECRET, + { + startIdempotency: startIdempotencyForEnv(env), + executionFence: executionFenceForEnv(env), + }, ), })(routed); if (agentResponse) return agentResponse; @@ -2768,6 +3044,11 @@ const handler: ExportedHandler = { status: runTopology.status, resume: runTopology.resume, terminate: runTopology.terminate, + startIdempotency: { + store: startIdempotencyForEnv(env), + live: runTopology.startLiveness, + executionFence: executionFenceForEnv(env), + }, })(routed); if (runResponse) return runResponse; diff --git a/packages/flowsafe/src/agent-host/index.ts b/packages/flowsafe/src/agent-host/index.ts index d06b5a4..af52ff3 100644 --- a/packages/flowsafe/src/agent-host/index.ts +++ b/packages/flowsafe/src/agent-host/index.ts @@ -48,6 +48,7 @@ export { type AgentThreadRunRef, type AgentThreadStartInput, type AgentThreadTopology, + type AgentThreadTopologyOptions, createAgentThreadTopology, } from './thread-topology.js'; export { diff --git a/packages/flowsafe/src/agent-host/router.test.ts b/packages/flowsafe/src/agent-host/router.test.ts index 5c1deff..7288d8f 100644 --- a/packages/flowsafe/src/agent-host/router.test.ts +++ b/packages/flowsafe/src/agent-host/router.test.ts @@ -12,7 +12,12 @@ import { humanPrincipal, InMemoryApprovalStoreFactory, } from '../approval-api/index.js'; -import { RunRouteError } from '../host-kit/index.js'; +import { + doErrorResponse, + type ExecutionFenceDatabase, + ExecutionFenceStore, +} from '../do-runner/index.js'; +import { doSummary, RunRouteError } from '../host-kit/index.js'; import { createAgentRouter } from './router.js'; import type { AgentThreadTopology } from './thread-topology.js'; import type { AgentRunEnvelope } from './types.js'; @@ -500,6 +505,121 @@ describe('createAgentRouter', () => { expect(host.observe).not.toHaveBeenCalled(); }); + it('passes a 5xx refusal through with its status and reason intact', async () => { + // #given — the run DO refusing because the deployment is fenced. This is a + // 5xx, which this router used to collapse into a bare 500 — turning "retry + // after the migration" into "I am broken" for every agent caller. + const host = topology(); + host.start.mockRejectedValueOnce( + new RunRouteError( + 503, + "deployment execution is fenced ('migration-locked'): run start is refused", + { code: 'EXECUTION_FENCED', state: 'migration-locked' }, + ), + ); + const router = createAgentRouter({ + agents, + resolve: async () => context(), + topology: host, + }); + + // #when + const response = await router( + new Request('https://host/agents/writer/runs', { + method: 'POST', + headers: { 'content-type': 'application/json' }, + body: JSON.stringify({ prompt: 'go' }), + }), + ); + + // #then + expect(response?.status).toBe(503); + expect(await response?.json()).toEqual({ + error: + "deployment execution is fenced ('migration-locked'): run start is refused", + reason: { code: 'EXECUTION_FENCED', state: 'migration-locked' }, + }); + }); + + it('passes an UNREADABLE fence through without the storage error behind it', async () => { + // #given — the whole real chain, not a hand-built error: a fence store over + // a database that faults with a message an operator would not want on the + // wire, the refusal that store raises, the DO's own error mapping, and the + // reader the topology uses to turn a DO response back into a RunRouteError. + const secret = 'D1_ERROR: connect ECONNREFUSED 10.0.7.4:5432 db=acme-prod'; + const fence = new ExecutionFenceStore({ + prepare: () => ({ + bind: () => ({ + run: () => Promise.reject(new Error(secret)), + all: () => Promise.reject(new Error(secret)), + }), + run: () => Promise.reject(new Error(secret)), + all: () => Promise.reject(new Error(secret)), + }), + } as unknown as ExecutionFenceDatabase); + const refusal = await fence.read().catch((error: unknown) => error); + const routeError = await doSummary(doErrorResponse(refusal)).catch( + (error: unknown) => error, + ); + const host = topology(); + host.start.mockRejectedValueOnce(routeError); + const router = createAgentRouter({ + agents, + resolve: async () => context(), + topology: host, + }); + + // #when + const response = await router( + new Request('https://host/agents/writer/runs', { + method: 'POST', + headers: { 'content-type': 'application/json' }, + body: JSON.stringify({ prompt: 'go' }), + }), + ); + + // #then — retryable and machine-readable, and the body is EXACTLY the two + // fields the taxonomy defines. The message-leak hazard is the passthrough + // itself: it forwards the DO's message verbatim at 5xx, so a refusal whose + // message carried the storage fault would put it in front of every agent + // caller. + expect(response?.status).toBe(503); + const body = await response?.text(); + expect(JSON.parse(body ?? '')).toEqual({ + error: 'execution fence state is not readable', + reason: { code: 'EXECUTION_FENCE_UNREADABLE' }, + }); + expect(body).not.toContain('ECONNREFUSED'); + expect(body).not.toContain('acme-prod'); + expect(body).not.toContain('10.0.7.4'); + }); + + it('still collapses a 5xx with NO structured reason into a generic 500', async () => { + // #given — the passthrough is narrow on purpose: only a code the DO + // deliberately published survives a 5xx; an unclassified fault must not + // start leaking its message to callers. + const host = topology(); + host.start.mockRejectedValueOnce( + new RunRouteError(502, 'upstream exploded with connection details'), + ); + const router = createAgentRouter({ + agents, + resolve: async () => context(), + topology: host, + }); + + // #then + const response = await router( + new Request('https://host/agents/writer/runs', { + method: 'POST', + headers: { 'content-type': 'application/json' }, + body: JSON.stringify({ prompt: 'go' }), + }), + ); + expect(response?.status).toBe(500); + expect(await response?.json()).toEqual({ error: 'internal server error' }); + }); + it('maps an asynchronously rejected observation to its route status', async () => { const host = topology(); host.observe.mockRejectedValue( diff --git a/packages/flowsafe/src/agent-host/router.ts b/packages/flowsafe/src/agent-host/router.ts index f82f43a..47507d4 100644 --- a/packages/flowsafe/src/agent-host/router.ts +++ b/packages/flowsafe/src/agent-host/router.ts @@ -6,7 +6,11 @@ import { RUN_START_ROLES, } from '../approval-api/index.js'; import { isPathSafeId } from '../do-runner/index.js'; -import { RunRouteError, requireResourceAccess } from '../host-kit/index.js'; +import { + RunRouteError, + requireResourceAccess, + runRouteReason, +} from '../host-kit/index.js'; import { readBoundedBody } from '../http-body.js'; import { createAgentCatalog } from './catalog.js'; import type { AgentThreadTopology } from './thread-topology.js'; @@ -184,7 +188,11 @@ function internalError(error: unknown, route: MatchedRoute): Response { return json({ error: 'forbidden' }, 403); } if (error instanceof RunRouteError) { - if (error.status < 500) { + // A structured reason is the DO saying what it refused and why, in a code + // it means to publish — so it passes through at ANY status, 5xx included. + // Without this a 503 EXECUTION_FENCED became a bare 500 here and the + // caller could not tell a fenced deployment from a broken one. + if (error.status < 500 || runRouteReason(error) !== undefined) { return json( { error: error.message, diff --git a/packages/flowsafe/src/agent-host/thread-host.test.ts b/packages/flowsafe/src/agent-host/thread-host.test.ts index e694ac8..026accd 100644 --- a/packages/flowsafe/src/agent-host/thread-host.test.ts +++ b/packages/flowsafe/src/agent-host/thread-host.test.ts @@ -1690,6 +1690,10 @@ describe('createThreadAgentHost', () => { 'operator-1', 'human', expect.any(String), + // The schedule dispatch and the reserved idempotency key: passed + // positionally on every start, and undefined on one that has neither. + undefined, + undefined, ); }); @@ -1747,7 +1751,9 @@ describe('createThreadAgentHost', () => { ); expect(response?.status).toBe(200); - expect(mocked.stream.mock.calls.at(-1)).toHaveLength(5); + // Five host arguments plus the two trailing optionals (schedule dispatch, + // reserved idempotency key), both undefined for this start. + expect(mocked.stream.mock.calls.at(-1)).toHaveLength(7); expect(discardScheduleDispatch).not.toHaveBeenCalled(); await expect( fixture.resources.owner('run', 'acme_run'), @@ -1996,6 +2002,8 @@ describe('createThreadAgentHost', () => { 'webhook-dispatcher', 'service', expect.any(String), + undefined, + undefined, ); await expect( fixture.resources.owner('run', 'acme_service_run'), @@ -2227,6 +2235,8 @@ describe('createThreadAgentHost', () => { 'signal-dispatcher', 'system', expect.any(String), + undefined, + undefined, ); await expect( fixture.resources.owner('run', 'acme_signal_wake'), diff --git a/packages/flowsafe/src/agent-host/thread-host.ts b/packages/flowsafe/src/agent-host/thread-host.ts index 816a505..19212d2 100644 --- a/packages/flowsafe/src/agent-host/thread-host.ts +++ b/packages/flowsafe/src/agent-host/thread-host.ts @@ -174,6 +174,17 @@ export interface ThreadAgentStartInput { safeContext?: Record; /** Trusted, JSON-safe model provider options from schedule dispatch only. */ providerOptions?: Record; + /** + * The idempotency key the thread topology already RESERVED for this run. + * + * Nothing at this layer reserves, claims, or replays on it: the reservation + * lives on the Worker side, where a retry that minted a fresh thread can + * still be redirected to the original one. It travels down here only so + * `RunnerRuntime.start` can compare it against the execution fence's + * nominated proof key. + * @internal + */ + idempotencyKey?: string; } export interface BoundThreadAgent { @@ -497,6 +508,20 @@ export function createThreadAgentHost( const withBindingLock = createFifoLock(); const withDispatchLock = createFifoLock(); const withRecoveryLock = createFifoLock(); + /** + * Run ids this object is currently starting — the liveness half of the + * idempotent-start replay decision, and the reason a retried agent start can + * tell "the first one is still working" from "the first one died holding the + * claim" without a timer. + * + * In memory, never stored: liveness is a property of an isolate that is + * running code, so the honest answer after an eviction is `false`, and any + * durable proxy would keep saying `true` for a run nothing is executing. + * Scoped to this host instance, which `instanceScopeFor` already pins to one + * Durable Object — the same object an agent run is bound to for its whole + * life, so this object's answer is the only one there is. + */ + const startsInFlight = new Set(); const instanceScopeFor = (scope: ThreadScope): AgentThreadInstanceScope => { if (stableScope) { @@ -1513,6 +1538,10 @@ export function createThreadAgentHost( await writeAgentRunRecord(options.stateStorage(), ref.runId, stored); return recovery; }); + // From here to the finally below, this object IS the run's execution. + // Registered BEFORE the stream so the window a replaying start asks + // about — the one before core has persisted anything — is covered too. + startsInFlight.add(ref.runId); try { const streamOptions = { runId: ref.runId, @@ -1544,7 +1573,8 @@ export function createThreadAgentHost( scope.principal.id, scope.principal.kind, recovery.token, - ...(scheduleDispatch ? ([scheduleDispatch] as const) : []), + scheduleDispatch, + input.idempotencyKey, ); const summary = await scope.init.runtime.status( DURABLE_AGENTIC_LOOP_WORKFLOW_ID, @@ -1613,6 +1643,11 @@ export function createThreadAgentHost( } } throw error; + } finally { + // Unconditional: an entry left behind would answer every later probe + // "live" for the lifetime of the isolate, turning a crashed start's + // honest UNRESOLVABLE into an endless PENDING. + startsInFlight.delete(ref.runId); } }); }, @@ -1648,6 +1683,25 @@ export function createThreadAgentHost( ? preflightUrl.pathname.slice(AGENT_HOST_ROUTE_PREFIX.length) : ''; const preflightSegments = preflightSuffix.split('/').filter(Boolean); + // The liveness probe, answered BEFORE withDispatchLock on purpose: the + // start it is asking about holds that lock for its whole first leg, so a + // probe that queued behind it would block for exactly as long as the run + // it was trying to describe — and time out reporting nothing. + // + // It reads no storage and reveals only whether this object is currently + // executing a run id the caller already had to know. Authorization is the + // deployment-identity header every request to this object carries: the + // probe travels the internal Worker-to-DO channel, and the reservation on + // the far side already proved the caller owns the key that names this run. + if ( + request.method === 'GET' && + preflightSegments.length === 4 && + preflightSegments[0] === 'runs' && + preflightSegments[3] === 'start-liveness' + ) { + const runId = decode(preflightSegments[2]); + return json({ live: runId !== undefined && startsInFlight.has(runId) }); + } if ( request.method === 'POST' && preflightSegments.length === 4 && @@ -1733,6 +1787,20 @@ export function createThreadAgentHost( threaded: body.threaded !== false, safeContext: safeContext(body.safeContext), providerOptions: providerOptions(body.providerOptions), + // Validated, not trusted: this body is JSON, and the same string + // reaches the execution fence's proof-only comparison. + ...(body.idempotencyKey === undefined + ? {} + : { + idempotencyKey: isPathSafeId(body.idempotencyKey) + ? body.idempotencyKey + : (() => { + throw new AgentHostRequestError( + 400, + 'idempotencyKey must be a URL-path-safe identifier', + ); + })(), + }), ...(body.scheduleId !== undefined ? { scheduleId: isPathSafeId(body.scheduleId) diff --git a/packages/flowsafe/src/agent-host/thread-topology.test.ts b/packages/flowsafe/src/agent-host/thread-topology.test.ts index ef9c8a9..60044a8 100644 --- a/packages/flowsafe/src/agent-host/thread-topology.test.ts +++ b/packages/flowsafe/src/agent-host/thread-topology.test.ts @@ -1,7 +1,12 @@ // SPDX-License-Identifier: Apache-2.0 import { describe, expect, it } from 'vitest'; +import { openSqlite, sqliteUnitDatabase } from '../../test-support/sqlite.js'; import type { ActorContext, ApprovalRecord } from '../approval-api/index.js'; +import { + type StartIdempotencyDatabase, + StartIdempotencyStore, +} from '../do-runner/index.js'; import type { ThreadNamespaceLike, ThreadRequestInit, @@ -78,7 +83,12 @@ function harness(): { }), }; return { - topology: createAgentThreadTopology(namespace, DEPLOYMENT_IDENTITY_SECRET), + // No database in this harness, so the opt-out is written down rather + // than defaulted — see AgentThreadTopologyOptions. + topology: createAgentThreadTopology(namespace, DEPLOYMENT_IDENTITY_SECRET, { + startIdempotency: 'none', + executionFence: 'none', + }), hits, }; } @@ -380,3 +390,437 @@ describe('createAgentThreadTopology', () => { }); }); }); + +// --------------------------------------------------------------------------- +// Owner-bound idempotent start, on the AGENT surface. +// +// The agent surface is where the reservation earns its keep. A workflow run has +// one possible host (idFromName(workflowId:runId)); an agent run lives in a +// thread object, and an unthreaded retry mints a FRESH thread every time — so +// without a recorded address a retry would be asking an empty object about a +// run it never had, and Durable Object serialization would not help because the +// two calls are two different objects. Every assertion counts STARTS. +// --------------------------------------------------------------------------- + +/** + * A thread namespace that behaves like a real one: each thread object holds its + * own runs, answers the liveness probe from its own in-flight set, and answers + * the dispatch status route only for runs it actually started. + */ +function keyedHarness(options: { now?: () => number } = {}) { + const sqlite = openSqlite(); + const store = new StartIdempotencyStore( + sqliteUnitDatabase(sqlite) as StartIdempotencyDatabase, + options.now ? { now: options.now } : {}, + ); + /** runId -> the thread that started it, i.e. where the run actually lives. */ + const runsByThread = new Map(); + const starts: Array<{ threadId: string; runId: string; key?: string }> = []; + const inFlight = new Set(); + const namespace: ThreadNamespaceLike = { + idFromName: (name) => name, + get: (threadId) => ({ + fetch: (async (request: Request | string, init?: ThreadRequestInit) => { + const url = typeof request === 'string' ? request : request.url; + if (url.includes('/start-liveness')) { + const runId = url.split('/runs/')[1]?.split('/')[1] ?? ''; + return Response.json({ + live: runsByThread.get(runId) === threadId && inFlight.has(runId), + }); + } + if (url.endsWith('/start')) { + const body = JSON.parse(init?.body ?? '{}') as Record; + runsByThread.set(body.runId as string, threadId); + inFlight.add(body.runId as string); + try { + await Promise.resolve(); + starts.push({ + threadId, + runId: body.runId as string, + ...(body.idempotencyKey === undefined + ? {} + : { key: body.idempotencyKey }), + }); + return Response.json({ + agentId: body.agentId, + threadId: body.threadId, + resourceId: body.resourceId, + runId: body.runId, + summary: { runId: body.runId, status: 'success' }, + }); + } finally { + inFlight.delete(body.runId as string); + } + } + // The dispatch status route: only the thread that started the run + // knows it. Every other object answers 404, which is exactly what a + // replay that guessed the wrong thread would get — and so does the + // owning thread while the run is still executing, because the first + // persisted summary lands only at the first suspend or terminal state. + const runId = url.split('/runs/')[1]?.split(/[/?]/)[1] ?? ''; + if (runsByThread.get(runId) !== threadId || inFlight.has(runId)) { + return Response.json({ error: 'run not found' }, { status: 404 }); + } + return Response.json({ + agentId: 'writer', + threadId, + resourceId: `acme_resource_${threadId}`, + runId, + summary: { runId, status: 'success' }, + }); + }) as ReturnType['get']>['fetch'], + }), + }; + return { + store, + starts, + runsByThread, + inFlight, + topology: createAgentThreadTopology(namespace, DEPLOYMENT_IDENTITY_SECRET, { + startIdempotency: store, + executionFence: 'none', + }), + }; +} + +describe('createAgentThreadTopology — idempotent start', () => { + it('refuses a key when the topology wired no reservation store', async () => { + // #given the typed opt-out + const { topology } = harness(); + + // #when / #then silently ignoring the key would answer an exactly-once + // request with at-least-once behaviour. + await expect( + topology.start(context().value, { + agentId: 'writer', + prompt: 'go', + entryPath: 'http.start', + idempotencyKey: 'key-1', + }), + ).rejects.toMatchObject({ + status: 503, + reason: { code: 'IDEMPOTENT_START_UNSUPPORTED' }, + }); + }); + + it('converges an UNTHREADED retry onto the original thread and run', async () => { + // #given the case DO serialization cannot cover: each call mints its own + // thread, so the two starts are two different objects + const { topology, starts } = keyedHarness(); + const scoped = context(); + const input = { + agentId: 'writer', + prompt: 'go', + entryPath: 'http.start' as const, + idempotencyKey: 'key-1', + }; + + // #when + const first = await topology.start(scoped.value, input); + const retry = await topology.start(scoped.value, input); + + // #then ONE start, and the retry answers with the original run on its + // original thread — the recorded address is the only thing that could have + // taken it back there. + expect(starts).toHaveLength(1); + expect(retry.runId).toBe(first.runId); + expect(retry.threadId).toBe(first.threadId); + }); + + it('routes a re-claimed reservation to the RECORDED thread, not the retry’s fresh one', async () => { + // #given a reservation an earlier caller left un-claimed on thread A — + // the crash window in which nothing has executed + const { topology, store, starts } = keyedHarness(); + const scoped = context(); + await store.reserve({ + key: 'key-1', + owner: { kind: 'human', id: 'operator-1' }, + targetKind: 'agent', + targetId: 'writer', + threadId: 'acme_thread_original', + mintRunId: () => 'acme_run_original', + }); + + // #when a retry arrives and mints a thread of its own + const result = await topology.start(scoped.value, { + agentId: 'writer', + prompt: 'go', + entryPath: 'http.start', + idempotencyKey: 'key-1', + }); + + // #then it starts on the RECORDED thread under the RESERVED run id. + // Starting on the freshly minted thread would put the run somewhere the + // reservation does not point, and the next retry could never find it. + expect(starts).toEqual([ + { + threadId: 'acme_thread_original', + runId: 'acme_run_original', + key: 'key-1', + }, + ]); + expect(result.runId).toBe('acme_run_original'); + }); + + it('carries the key on the internal channel so the fence can match it', async () => { + // #given + const { topology, starts } = keyedHarness(); + + // #when + await topology.start(context().value, { + agentId: 'writer', + prompt: 'go', + entryPath: 'http.start', + idempotencyKey: 'key-1', + }); + + // #then the key reaches the thread object's start body, which is where + // RunnerRuntime.start reads it from for the proof-only comparison. + expect(starts[0]?.key).toBe('key-1'); + }); + + it('starts ONE run for two same-key calls issued in parallel', async () => { + // #given two in-flight first calls on one key, neither having seen the + // other, each minting its own thread + const { topology, starts } = keyedHarness(); + const scoped = context(); + const input = { + agentId: 'writer', + prompt: 'go', + entryPath: 'http.start' as const, + idempotencyKey: 'key-1', + }; + + // #when + const outcomes = await Promise.allSettled([ + topology.start(scoped.value, input), + topology.start(scoped.value, input), + ]); + + // #then exactly one start. The loser may replay, or be refused as PENDING, + // or hit the claim-to-dispatch window and be refused as UNRESOLVABLE — + // what it must never do is produce a second run. + expect(starts).toHaveLength(1); + expect(outcomes).toHaveLength(2); + }); + + it('refuses a key reused for a different agent', async () => { + // #given + const { topology, starts } = keyedHarness(); + const scoped = context(); + await topology.start(scoped.value, { + agentId: 'writer', + prompt: 'go', + entryPath: 'http.start', + idempotencyKey: 'key-1', + }); + + // #when + const refusal = await topology + .start(scoped.value, { + agentId: 'editor', + prompt: 'go', + entryPath: 'http.start', + idempotencyKey: 'key-1', + }) + .catch((error: unknown) => error); + + // #then a key that meant one agent cannot come to mean another + expect(refusal).toMatchObject({ + status: 409, + reason: { + code: 'IDEMPOTENT_START_TARGET_MISMATCH', + targetId: 'writer', + }, + }); + expect(starts).toHaveLength(1); + }); + + it('refuses a key another principal reserved', async () => { + // #given a reservation held by someone else + const { topology, store } = keyedHarness(); + await store.reserve({ + key: 'key-1', + owner: { kind: 'human', id: 'operator-2' }, + targetKind: 'agent', + targetId: 'writer', + threadId: 'acme_thread_other', + mintRunId: () => 'acme_run_other', + }); + + // #when / #then + await expect( + topology.start(context().value, { + agentId: 'writer', + prompt: 'go', + entryPath: 'http.start', + idempotencyKey: 'key-1', + }), + ).rejects.toMatchObject({ + status: 403, + reason: { code: 'IDEMPOTENT_START_OWNER_MISMATCH' }, + }); + }); + + it('probes the RECORDED thread for liveness and reports PENDING', async () => { + // #given a claimed reservation whose run is executing on its own thread + const { topology, store, runsByThread, inFlight } = keyedHarness(); + await store.reserve({ + key: 'key-1', + owner: { kind: 'human', id: 'operator-1' }, + targetKind: 'agent', + targetId: 'writer', + threadId: 'acme_thread_live', + mintRunId: () => 'acme_run_live', + }); + await store.claim('key-1', 'acme_run_live'); + runsByThread.set('acme_run_live', 'acme_thread_live'); + inFlight.add('acme_run_live'); + + // #when a retry arrives, minting a different thread of its own + const refusal = await topology + .start(context().value, { + agentId: 'writer', + prompt: 'go', + entryPath: 'http.start', + idempotencyKey: 'key-1', + }) + .catch((error: unknown) => error); + + // #then 503 PENDING — the probe found the run alive on the thread the + // RESERVATION recorded, which is not the thread this retry minted. + expect(refusal).toMatchObject({ + status: 503, + reason: { code: 'IDEMPOTENT_START_PENDING', runId: 'acme_run_live' }, + }); + }); + + it('reports UNRESOLVABLE when the recorded thread is not running the claim', async () => { + // #given a claim held by a thread object that is executing nothing + const { topology, store } = keyedHarness(); + await store.reserve({ + key: 'key-1', + owner: { kind: 'human', id: 'operator-1' }, + targetKind: 'agent', + targetId: 'writer', + threadId: 'acme_thread_dead', + mintRunId: () => 'acme_run_dead', + }); + await store.claim('key-1', 'acme_run_dead'); + + // #when / #then never re-executed: whether the agent's first tool call + // already fired is unknowable from here. + await expect( + topology.start(context().value, { + agentId: 'writer', + prompt: 'go', + entryPath: 'http.start', + idempotencyKey: 'key-1', + }), + ).rejects.toMatchObject({ + status: 409, + reason: { code: 'IDEMPOTENT_START_UNRESOLVABLE' }, + }); + }); + + it('reports ALREADY_SETTLED for a spent key whose run aged out', async () => { + // #given + const { topology, store } = keyedHarness(); + await store.reserve({ + key: 'key-1', + owner: { kind: 'human', id: 'operator-1' }, + targetKind: 'agent', + targetId: 'writer', + threadId: 'acme_thread_gone', + mintRunId: () => 'acme_run_gone', + }); + await store.claim('key-1', 'acme_run_gone'); + await store.settleRun('acme_run_gone'); + + // #when / #then + await expect( + topology.start(context().value, { + agentId: 'writer', + prompt: 'go', + entryPath: 'http.start', + idempotencyKey: 'key-1', + }), + ).rejects.toMatchObject({ + status: 409, + reason: { code: 'IDEMPOTENT_START_ALREADY_SETTLED' }, + }); + }); + + it('gives the claim back when the thread object reports the fence closed', async () => { + // #given a thread object refusing the start with the fence's own code — + // rebuilt from a DO response, so no longer an ExecutionFencedError instance + const sqlite = openSqlite(); + const store = new StartIdempotencyStore( + sqliteUnitDatabase(sqlite) as StartIdempotencyDatabase, + ); + const namespace: ThreadNamespaceLike = { + idFromName: (name) => name, + get: () => ({ + fetch: (async (request: Request | string, init?: ThreadRequestInit) => { + const url = typeof request === 'string' ? request : request.url; + if (url.includes('/start-liveness')) { + return Response.json({ live: false }); + } + if (url.endsWith('/start')) { + return Response.json( + { + error: "deployment execution is fenced ('migration-locked')", + reason: { + code: 'EXECUTION_FENCED', + state: 'migration-locked', + }, + }, + { status: 503 }, + ); + } + void init; + return Response.json({ error: 'run not found' }, { status: 404 }); + }) as ReturnType['get']>['fetch'], + }), + }; + const topology = createAgentThreadTopology( + namespace, + DEPLOYMENT_IDENTITY_SECRET, + { startIdempotency: store, executionFence: 'none' }, + ); + + // #when + const refusal = await topology + .start(context().value, { + agentId: 'writer', + prompt: 'go', + entryPath: 'http.start', + idempotencyKey: 'key-1', + }) + .catch((error: unknown) => error); + + // #then the fence's refusal reached the caller AND the claim went back, so + // a retry after the operator reopens converges on the same run instead of + // finding a key poisoned by a drain. + expect(refusal).toMatchObject({ reason: { code: 'EXECUTION_FENCED' } }); + expect((await store.read('key-1'))?.state).toBe('reserved'); + }); + + it('leaves an unkeyed start byte-identical to before the reservation existed', async () => { + // #given a wired topology + const { topology, store, starts } = keyedHarness(); + + // #when a start arrives with no key + await topology.start(context().value, { + agentId: 'writer', + prompt: 'go', + entryPath: 'http.start', + }); + + // #then it started, and reserved nothing: a host that wires the store does + // not thereby make every start pay for a table it never asked for. + expect(starts).toHaveLength(1); + expect(starts[0]?.key).toBeUndefined(); + expect(await store.read('key-1')).toBeUndefined(); + }); +}); diff --git a/packages/flowsafe/src/agent-host/thread-topology.ts b/packages/flowsafe/src/agent-host/thread-topology.ts index 7109b30..1d56d23 100644 --- a/packages/flowsafe/src/agent-host/thread-topology.ts +++ b/packages/flowsafe/src/agent-host/thread-topology.ts @@ -6,7 +6,15 @@ import { type ApprovalRecord, defaultResumeData, } from '../approval-api/index.js'; -import { isPathSafeId } from '../do-runner/index.js'; +import { + beginIdempotentStart, + type ExecutionFenceWiring, + isPathSafeId, + requireStartIdempotency, + rollbackFencedStart, + type StartIdempotencyWiring, + type StartReservation, +} from '../do-runner/index.js'; import { type BoundThreadTarget, type BoundThreadTargetValidator, @@ -47,6 +55,40 @@ export interface AgentThreadStartInput { scheduleId?: string; /** Prepared schedule trigger authorizing this exact run. */ dispatchId?: string; + /** + * Makes this start exactly-once for this principal: a retry carrying the same + * key converges onto the run the first call made, on whatever thread it made + * it, instead of starting a second one. + * + * TRUSTED-CALLER ONLY, and not by omission: the public agent router rejects + * every body field but `prompt`, so a key can only arrive from a host seam — + * a schedule adapter, a delegating agent, a host's own start path. That is + * the same posture `requestContext` and `providerOptions` take here, and for + * the same reason: a tenant able to name a key could converge onto, or + * collide with, a run it did not make. (Ownership makes the collision refuse + * rather than succeed, but a tenant-reachable key would still be a probe.) + */ + idempotencyKey?: string; +} + +/** + * How the agent topology honours `idempotencyKey`, or the typed opt-out. + * + * REQUIRED on `createAgentThreadTopology` for the same reason the run router's + * is: a topology that silently ignored a key would answer an exactly-once + * request with at-least-once behaviour, and the caller would have no way to + * find out. Writing `'none'` is honest — it makes every keyed start refuse with + * IDEMPOTENT_START_UNSUPPORTED, and leaves unkeyed starts byte-identical. + */ +export interface AgentThreadTopologyOptions { + startIdempotency: StartIdempotencyWiring; + /** + * The deployment execution fence, so a REPLAY can re-assert a proof-only + * fence's binding to the run this key already made. Must be the fence over + * the SAME database the reservations live in; `'none'` is the honest answer + * for a host with no fence. + */ + executionFence: ExecutionFenceWiring; } export interface AgentThreadRunRef { @@ -155,6 +197,7 @@ function expectedResource( export function createAgentThreadTopology( namespace: ThreadNamespaceLike, deploymentIdentitySecret: string, + options: AgentThreadTopologyOptions, ): AgentThreadDispatchTopology { const threads: ThreadTopology = createThreadTopology( namespace, @@ -206,6 +249,78 @@ export function createAgentThreadTopology( const resourceId = expectedResource(context, input.threadId); return fetchStatus(context, input, resourceId, true); }; + /** + * A reserved agent run's persisted envelope, read from the thread the + * reservation recorded — which is NOT necessarily the thread this retry would + * have minted. That indirection is the whole reason the reservation stores a + * thread at all: an unthreaded retry mints a fresh thread every time, and + * without the recorded one it would be asking an empty object about a run it + * never had. + * + * The dispatch variant of the status route, because a run that already + * reached a terminal state has had its run record and ownership released, and + * the public variant answers 404 for exactly that case — which a replay must + * not confuse with "no run". + * + * NO `requireResourceAccess` here, deliberately. Ownership on this path is + * the RESERVATION's: the reserve call already refused every principal but the + * key's owner, and the resource registry it would consult has legitimately + * forgotten a settled run. Re-checking there would make a completed run's + * replay a 404 — the one answer that would send a caller off to start a + * second one. + */ + const reservedRunEnvelope = async ( + context: ActorContext, + agentId: string, + reservation: StartReservation, + ): Promise => { + const threadId = reservation.threadId; + if (threadId === undefined || !isPathSafeId(threadId)) return undefined; + const response = await threads.send( + context, + threadId, + `${AGENT_HOST_ROUTE_PREFIX}/runs/${encodeURIComponent( + agentId, + )}/${encodeURIComponent( + reservation.runId, + )}?resourceId=${encodeURIComponent( + context.resourceIdFromKey(threadId), + )}&dispatch=1`, + ); + if (response.status === 404) return undefined; + return envelope(response); + }; + /** + * Is the reserved run executing in its thread object right now? + * + * Asked of the RECORDED thread, which is the only object that could be + * running it: an agent run is bound to one thread for its whole life. An + * unreachable or unparseable answer reads as NOT live, the fail-closed + * direction here — it produces the refusal that asks a human to investigate, + * where a default of "live" would answer a permanently dead run with a + * permanently retryable 503. + */ + const reservedRunLive = async ( + context: ActorContext, + agentId: string, + reservation: StartReservation, + ): Promise => { + const threadId = reservation.threadId; + if (threadId === undefined || !isPathSafeId(threadId)) return false; + const response = await threads.send( + context, + threadId, + `${AGENT_HOST_ROUTE_PREFIX}/runs/${encodeURIComponent( + agentId, + )}/${encodeURIComponent(reservation.runId)}/start-liveness`, + ); + if (response.status !== 200) return false; + try { + return ((await response.json()) as { live?: unknown }).live === true; + } catch { + return false; + } + }; return { requireBoundThread: async (context, target: BoundThreadTarget) => { const threadId = validThread(target.threadId); @@ -266,7 +381,13 @@ export function createAgentThreadTopology( ? context.newThreadId() : validThread(input.threadId); const resourceId = expectedResource(context, threadId, input.resourceId); - const runId = + // The mint, as a THUNK. Two things depend on the delay: the reservation + // only spends an id when it wins the insert, and — for the + // caller-influenced `input.runId` path this closes the gap on — a retry + // that loses the insert takes the WINNER's id rather than re-proposing + // its own, which is what makes two same-key starts converge instead of + // becoming two runs. + const mintRunId = (): string => input.runId === undefined ? context.newRunId() : isPathSafeId(input.runId) @@ -294,34 +415,90 @@ export function createAgentThreadTopology( ...input.requestContext, ...input.streamRequestContext, }); - return envelope( - await threads.send( - context, - threadId, - `${AGENT_HOST_ROUTE_PREFIX}/start`, - { - method: 'POST', - headers: { 'content-type': 'application/json' }, - body: JSON.stringify({ - agentId: input.agentId, - threadId, - resourceId, - runId, - prompt: input.prompt, - entryPath: input.entryPath, - threaded, - safeContext, - providerOptions: input.providerOptions, - ...(input.scheduleId !== undefined - ? { scheduleId: input.scheduleId } - : {}), - ...(input.dispatchId !== undefined - ? { dispatchId: input.dispatchId } - : {}), - }), + const sendStart = async ( + targetThreadId: string, + targetRunId: string, + idempotencyKey?: string, + ): Promise => + envelope( + await threads.send( + context, + targetThreadId, + `${AGENT_HOST_ROUTE_PREFIX}/start`, + { + method: 'POST', + headers: { 'content-type': 'application/json' }, + body: JSON.stringify({ + agentId: input.agentId, + threadId: targetThreadId, + resourceId: context.resourceIdFromKey(targetThreadId), + runId: targetRunId, + prompt: input.prompt, + entryPath: input.entryPath, + threaded, + safeContext, + providerOptions: input.providerOptions, + // The key rides the internal Worker-to-DO channel only, so the + // fence's proof-only state can match it inside the runtime. + ...(idempotencyKey === undefined ? {} : { idempotencyKey }), + ...(input.scheduleId !== undefined + ? { scheduleId: input.scheduleId } + : {}), + ...(input.dispatchId !== undefined + ? { dispatchId: input.dispatchId } + : {}), + }), + }, + ), + ); + if (input.idempotencyKey === undefined) { + // Unkeyed starts take the path they always took, on the thread this + // call resolved. + return sendStart(threadId, mintRunId()); + } + const store = requireStartIdempotency(options.startIdempotency); + const decision = await beginIdempotentStart( + store, + { + key: input.idempotencyKey, + owner: { + kind: context.principal.kind, + id: context.principal.id, }, - ), + targetKind: 'agent', + targetId: input.agentId, + threadId, + mintRunId, + }, + { + persisted: (reservation) => + reservedRunEnvelope(context, input.agentId, reservation), + live: (reservation) => + reservedRunLive(context, input.agentId, reservation), + }, + options.executionFence, ); + if (decision.kind === 'replay') return decision.persisted; + // The RESERVATION's thread and run, not this call's: on a re-claim of a + // reservation an earlier crashed caller left behind, the recorded thread + // is where that run belongs and this call's freshly minted one is not. + const { reservation } = decision; + const startThreadId = reservation.threadId ?? threadId; + try { + return await sendStart( + startThreadId, + reservation.runId, + reservation.key, + ); + } catch (error) { + // Only a fence refusal gives the claim back — see rollbackFencedStart. + return rollbackFencedStart( + store, + reservation.key, + reservation.runId, + error, + ); + } }, status: statusFromHost, dispatchStatus: dispatchStatusFromHost, diff --git a/packages/flowsafe/src/agent-runner/agent-gate-round-trip.test.ts b/packages/flowsafe/src/agent-runner/agent-gate-round-trip.test.ts index cf45633..7d59bfc 100644 --- a/packages/flowsafe/src/agent-runner/agent-gate-round-trip.test.ts +++ b/packages/flowsafe/src/agent-runner/agent-gate-round-trip.test.ts @@ -1,19 +1,19 @@ // SPDX-License-Identifier: Apache-2.0 -// Track A acceptance criteria #2 + #3, proven against the REAL runtime + -// breakwater connector + grant provider + host-kit bridge (no LLM needed): +// The durable-agent approval round trip, proven against the REAL runtime, +// breakwater connector, grant provider, and host-kit bridge (no LLM needed): // -// - S1/S2: the ENGINE-LEG requestContext reaches the connector, so an approved -// agent gate mints the grant the write gate demands and the run completes; -// a FORGED resume that mints no grant fails closed at that same gate. -// - R-003: BOTH durable approval-suspend shapes (flat + nested) round-trip +// - The ENGINE-LEG requestContext reaches the connector, so an approved agent +// gate mints the grant the write gate demands and the run completes. A +// FORGED resume that mints no grant fails closed at that same gate. +// - BOTH durable approval-suspend shapes (flat + nested) round-trip // through the grant-only path — the bridge derives connectors:[toolName] // from the agent shape (not an explicit `connectors` array), and the // decision resumes on the (suspendedAt, resumeCount) fingerprint. // -// This is the composition S1 verified in dist made executable: the workflow -// here MIMICS the durable-agentic-loop's tool-call gate (suspend with the agent -// payload shape, then call a write-gated connector using the step-param -// requestContext), which is exactly the mechanic the real loop uses +// This makes the dist-verified composition executable: the workflow MIMICS the +// durable-agentic-loop's tool-call gate by suspending with the agent payload +// shape, then calling a write-gated connector with the step-param +// requestContext. This is exactly the mechanic the real loop uses // (agent/durable index.js: params.requestContext -> toolOptions -> tool.execute). import { Agent } from '@mastra/core/agent'; @@ -161,6 +161,8 @@ function buildHarness( const { createWorkflow, createStep, runtime } = init( { storage }, { + startIdempotency: 'none', + executionFence: 'none', requestContextForRun: async (id, runId, leg) => ({ ...(await grantProvider(id, runId, leg)), ...(options.principalPermissions !== undefined @@ -247,6 +249,9 @@ function buildHarness( const runtime = makeRuntime(); const service = new ApprovalService({ store, + // In-memory store, no database to fence against: the opt-out is written down + // rather than defaulted — see ExecutionFenceWiring. + executionFence: 'none', resumeRun: resumeViaRuntime(runtime), }); @@ -261,7 +266,7 @@ function buildHarness( }; } -describe('agent gate grant round-trip (R-003, both shapes)', () => { +describe('agent gate grant round-trip (both suspension shapes)', () => { for (const shape of ['flat', 'nested'] as const) { it(`${shape} shape: an approved agent gate mints the grant and the connector runs`, async () => { // #given — a run suspended at the agent tool-call gate @@ -273,8 +278,8 @@ describe('agent gate grant round-trip (R-003, both shapes)', () => { expect(started.status).toBe('suspended'); // #when — the suspension is bridged to an approval record. The bridge must - // derive the connector to grant FROM the agent suspend shape (R-003), - // since the payload carries no explicit `connectors` array. + // derive the connector to grant FROM the agent suspend shape, since the + // payload carries no explicit `connectors` array. const [record] = await queueApprovalForSuspension( h.service, 'agent-launch', @@ -525,6 +530,7 @@ describe('agent gate grant round-trip (R-003, both shapes)', () => { const evicted = h.makeRuntime(); const service = new ApprovalService({ store: h.store, + executionFence: 'none', resumeRun: resumeViaRuntime(evicted), }); const [record] = await queueApprovalForSuspension( @@ -560,8 +566,14 @@ describe('agent gate grant round-trip (R-003, both shapes)', () => { const store = new InMemoryApprovalStore(); const connectorAudit = new AuditLogger(); const makeRuntime = () => - init({ storage }, { requestContextForRun: approvalGrantProvider(store) }) - .runtime; + init( + { storage }, + { + startIdempotency: 'none', + requestContextForRun: approvalGrantProvider(store), + executionFence: 'none', + }, + ).runtime; const modelCalls = vi.fn(); let initialPublishes = 0; let rehydratedPublishes = 0; @@ -647,6 +659,7 @@ describe('agent gate grant round-trip (R-003, both shapes)', () => { } as never); const service = new ApprovalService({ store, + executionFence: 'none', resumeRun: (record, decision) => reconstructed.resumeViaRuntime({ runId: record.runId, diff --git a/packages/flowsafe/src/agent-runner/durable-agent-runner.ts b/packages/flowsafe/src/agent-runner/durable-agent-runner.ts index a66cfac..937d20b 100644 --- a/packages/flowsafe/src/agent-runner/durable-agent-runner.ts +++ b/packages/flowsafe/src/agent-runner/durable-agent-runner.ts @@ -1,13 +1,13 @@ // SPDX-License-Identifier: Apache-2.0 // FlowsafeDurableAgent — drive Mastra's durable-agent loop through the ONE -// RunnerRuntime chokepoint (DL-001/DL-010) so every agent leg inherits the -// substrate's invariants: INV-1 server-minted runIds, per-leg -// requestContextForRun grant derivation, snapshot provenance, RunSummary, and -// retention purge — with no second execution path to audit. +// RunnerRuntime chokepoint so every agent leg inherits the substrate's +// invariants: server-minted runIds, per-leg requestContextForRun grant +// derivation, snapshot provenance, RunSummary, and retention purge, with no +// second execution path to audit. // -// The mechanic (validated against @mastra/core 1.50.0 dist, spike S1 — every -// offset in THIS section is 1.50.0-vintage and deliberately kept as the -// provenance of the original validation; later sections carry their own stamp): +// The mechanic was validated against @mastra/core 1.50.0 dist. Every offset in +// THIS section is 1.50.0-vintage and deliberately kept as the provenance of the +// original validation; later sections carry their own stamp: // DurableAgent compiles the agent loop to the default-engine workflow // 'durable-agentic-loop' (agent/durable index.js: AGENTIC_LOOP :62, // getWorkflow() :5936, agent-agnostic — the agent is resolved per run by the @@ -30,20 +30,20 @@ // #requestContextFor mints per leg — so approvalGrantProvider's // `breakwater.connectorGrants` grant reaches the connector write gate with // zero extra wiring, and a forged/self resume that mints no grant fails closed -// there (the registry copy is read only for the fail-closed, over-require-safe -// approval PRE-check, S5). +// there. The registry copy is read only for the fail-closed, over-require-safe +// approval pre-check. // -// INV-1 at the boundary: stream()/generate()/prepare() are the THREE inherited -// minting entry points — each takes an OPTIONAL runId and, when it is absent, -// lets core mint an unowned crypto.randomUUID() upstream +// Host-owned run ids at the boundary: stream()/generate()/prepare() are the +// THREE inherited minting entry points. Each takes an OPTIONAL runId and, when +// it is absent, lets core mint an unowned crypto.randomUUID() upstream // (prepareForDurableExecution, agent/durable index.js:589) that // PATH_SAFE_ID_PATTERN then accepts, slipping past executeWorkflow's guard AND -// RunnerRuntime.start's (the exact fallback INV-1 forbids). All three are -// overridden ONLY to REQUIRE a caller-minted runId before delegating to super — -// and prepare() also REGISTERS the run under that id (index.js:5984), so an -// unguarded prepare() strands an unowned run in the registry. streamUntilIdle() -// needs no override: it drives agent.stream() (index.js:368), so the stream() -// guard already covers it. +// RunnerRuntime.start's exact fallback, which the host-owned run-id rule +// forbids. All three are overridden ONLY to require a caller-minted runId +// before delegating to super. prepare() also REGISTERS the run under that id +// (index.js:5984), so an unguarded prepare() strands an unowned run in the +// registry. streamUntilIdle() needs no override: it drives agent.stream() +// (index.js:368), so the stream() guard already covers it. // // EVERY OTHER inherited entry point that can drive a run — or that hands back // runs the caller may not own — is BLOCKED, overridden to throw before it @@ -206,8 +206,9 @@ // member is harmless there, while an INSTANCE Mastra calls in-process must // throw. // -// Blocking them keeps A-D2/P8 true by construction: resumeViaRuntime() is the -// ONLY way a run resumes. ApprovalService.decide -> the host's ResumeRunFn -> +// Blocking them keeps the single-resume and no-capability guarantees true by +// construction: resumeViaRuntime() is the ONLY way a run resumes. +// ApprovalService.decide -> the host's ResumeRunFn -> // createAgentApprovalResumer, which hands every 'durable-agentic-loop' record to // the thread topology's resume (-> resumeViaRuntime -> runtime.resume) and // refuses one carrying no agent-thread target rather than falling through to the @@ -220,7 +221,7 @@ // // Live-isolate scope: the loop resolves the tool's execute closure from the // in-process globalRunRegistry (populated by stream()). A DO holds one run in -// one isolate (P1), so a resume decided before eviction finds it. A resume +// one isolate, so a resume decided before eviction finds it. A resume // AFTER eviction must first rehydrate that registry without replaying // application input processors. resumeViaRuntime() rebuilds the registry with // complete runtime processor lists after invoking only reserved RBAC during @@ -406,9 +407,9 @@ const THREAD_TOOL_APPROVAL_REASON = */ export const BLOCKED_RUN_ENTRIES = { recover: - 're-driving a persisted run bypasses run ownership (INV-1), per-leg grant minting and the fail-closed registry rehydration', + 're-driving a persisted run bypasses run ownership, per-leg grant minting and the fail-closed registry rehydration', recoverActiveRuns: - 'bulk re-driving persisted runs bypasses run ownership (INV-1) and per-leg grant minting', + 'bulk re-driving persisted runs bypasses run ownership and per-leg grant minting', resume: RESUME_FAMILY_REASON, resumeStream: RESUME_FAMILY_REASON, resumeGenerate: RESUME_FAMILY_REASON, @@ -422,7 +423,7 @@ export const BLOCKED_RUN_ENTRIES = { "core scopes the suspended-run listing by agentId plus the caller's own optional thread and resource ids, never by per-principal ownership, so it bypasses the host topology's run-ownership checks and returns run, thread and resource ids the caller does not own", deleteRunSnapshots: 'durable-agent snapshot rows are retained until deployment-scoped retention purge removes them', - network: `${NETWORK_FAMILY_REASON}, and it mints an unowned run id when the caller omits one (INV-1)`, + network: `${NETWORK_FAMILY_REASON}, and it mints an unowned run id when the caller omits one`, resumeNetwork: NETWORK_FAMILY_REASON, approveNetworkToolCall: NETWORK_FAMILY_REASON, declineNetworkToolCall: NETWORK_FAMILY_REASON, @@ -573,6 +574,14 @@ export class FlowsafeDurableAgent< readonly #startRequesters = new Map(); readonly #startRequesterKinds = new Map(); readonly #startAttemptTokens = new Map(); + /** + * runId -> the reservation key the thread topology took for this start, so + * `executeWorkflow` can hand it to `RunnerRuntime.start` for the execution + * fence's proof-only match. Same lifetime and same cleanup as the attempt + * token beside it: registered by `streamUntilPersisted`, dropped in its + * `finally`, so nothing outlives the start it belongs to. + */ + readonly #startIdempotencyKeys = new Map(); readonly #startScheduleDispatches = new Map< string, { scheduleId: string; dispatchId: string } @@ -587,8 +596,8 @@ export class FlowsafeDurableAgent< id: options.id, name: options.name, cache: options.cache, - // Default the agent's own stream pubsub to the runtime's identity (DL-001: - // ONE feed per DO), so a host that configures only init()'s pubsub still + // Default the agent's own stream pubsub to the runtime's identity (ONE + // feed per DO), so a host that configures only init()'s pubsub still // gets observe()/emitError aligned with the run's events (the run is driven // through the runtime, which publishes on THAT identity). An explicit // pubsub wins; both-absent falls to core's per-agent default (poll-only, @@ -621,7 +630,7 @@ export class FlowsafeDurableAgent< #assertCallerRunId(runId: unknown): asserts runId is string { if (!isPathSafeId(runId)) { throw new InvalidRunRequestError( - 'a caller-minted runId is required and must be URL-path-safe (INV-1: the host owns run ids) — the durable-agent runner never generates one', + 'a caller-minted runId is required and must be URL-path-safe (the host owns run ids) — the durable-agent runner never generates one', ); } } @@ -722,6 +731,21 @@ export class FlowsafeDurableAgent< requestedByKind: ExecutionPrincipalKind, attemptToken = crypto.randomUUID(), scheduleDispatch?: { scheduleId: string; dispatchId: string }, + /** + * The idempotency key the thread topology already RESERVED for this run. + * + * It travels down here for one job: the execution fence's proof-only state + * admits exactly the start whose key matches its nominated proof key, and + * `RunnerRuntime.start` is where that comparison happens. It buys no + * exactly-once property at this layer — the reservation above already did — + * so nothing here validates it beyond passing it on, and a run started + * without one behaves exactly as before. + * + * Parked per-runId beside the attempt token rather than threaded through + * core, because core owns the call between `stream()` and + * `executeWorkflow()` and carries no field this could ride in. + */ + idempotencyKey?: string, ): Promise< Awaited['stream']>> > { @@ -755,6 +779,9 @@ export class FlowsafeDurableAgent< if (scheduleDispatch) { this.#startScheduleDispatches.set(runId, scheduleDispatch); } + if (idempotencyKey !== undefined) { + this.#startIdempotencyKeys.set(runId, idempotencyKey); + } const onError = callOptions.onError; try { const hostCallOptions: typeof callOptions = { @@ -778,6 +805,7 @@ export class FlowsafeDurableAgent< this.#startRequesterKinds.delete(runId); this.#startAttemptTokens.delete(runId); this.#startScheduleDispatches.delete(runId); + this.#startIdempotencyKeys.delete(runId); } } @@ -1476,11 +1504,13 @@ export class FlowsafeDurableAgent< const requestedByKind = this.#startRequesterKinds.get(runId); const attemptToken = this.#startAttemptTokens.get(runId); const scheduleDispatch = this.#startScheduleDispatches.get(runId); + const idempotencyKey = this.#startIdempotencyKeys.get(runId); const startOptions = { runId, inputData: workflowInput, ...(attemptToken === undefined ? {} : { attemptToken }), ...(scheduleDispatch === undefined ? {} : { scheduleDispatch }), + ...(idempotencyKey === undefined ? {} : { idempotencyKey }), }; if (requestedBy === undefined || requestedByKind === undefined) { if (requestedBy !== undefined || requestedByKind !== undefined) { diff --git a/packages/flowsafe/src/approval-api/d1-store.ts b/packages/flowsafe/src/approval-api/d1-store.ts index a5b59e5..cf49dd2 100644 --- a/packages/flowsafe/src/approval-api/d1-store.ts +++ b/packages/flowsafe/src/approval-api/d1-store.ts @@ -19,6 +19,7 @@ import { stepKeyOf, } from './store.js'; import { + APPROVALS_TABLE, type ApprovalListFilter, type ApprovalMetrics, type ApprovalRecord, @@ -48,7 +49,10 @@ export interface ApprovalPreparedStatement { all(): Promise<{ results: T[] }>; } -const TABLE = 'flowsafe_approvals'; +// The name itself lives on the types leaf, which is the only approval-api file +// the drain inventory is allowed to reach. This is the local shorthand every +// query below was already written against. +const TABLE = APPROVALS_TABLE; // The partial index preserves open-step uniqueness. Captured suspensions add a // second, all-status fingerprint index after the legacy nullable columns are @@ -706,7 +710,7 @@ export class D1ApprovalStore implements ApprovalStore { const values: unknown[] = []; appendListFilters(filter, where, values); const clause = where.length > 0 ? ` WHERE ${where.join(' AND ')}` : ''; - // D3: default a bare list() to the max, so a repeated poll can never fall + // Bound a bare list() to the maximum, so a repeated poll can never fall // back to an unbounded SELECT. const limit = clampApprovalLimit(filter.limit) ?? MAX_APPROVAL_LIST_LIMIT; const { results } = await this.#db diff --git a/packages/flowsafe/src/approval-api/end-to-end.test.ts b/packages/flowsafe/src/approval-api/end-to-end.test.ts index f362222..afaf7e1 100644 --- a/packages/flowsafe/src/approval-api/end-to-end.test.ts +++ b/packages/flowsafe/src/approval-api/end-to-end.test.ts @@ -1,5 +1,5 @@ // SPDX-License-Identifier: Apache-2.0 -// Cross-package proof of the Phase 3 loop. breakwater is a devDependency +// Cross-package proof of the approval loop. breakwater is a devDependency // resolved FROM SOURCE (vitest alias + tsconfig.test paths), so `pnpm -r // test` never needs a built breakwater dist. // @@ -267,7 +267,11 @@ function buildHarness(): Harness { const { createWorkflow, createStep, runtime } = init( { storage: new InMemoryStore() }, - { requestContextForRun: approvalGrantProvider(store) }, + { + startIdempotency: 'none', + requestContextForRun: approvalGrantProvider(store), + executionFence: 'none', + }, ); const research = createStep({ @@ -451,6 +455,9 @@ function buildHarness(): Harness { const service = new ApprovalService({ store, + // In-memory store, no database to fence against: the opt-out is written down + // rather than defaulted — see ExecutionFenceWiring. + executionFence: 'none', resumeRun: resumeViaRuntime(runtime), }); @@ -498,6 +505,7 @@ describe('fail closed: the HTTP create route cannot mint a run-scoped standing g ) { const service = new ApprovalService({ store, + executionFence: 'none', ...(options.resumeRun === undefined ? {} : { resumeRun: options.resumeRun }), @@ -731,7 +739,7 @@ describe('fail closed: the HTTP create route cannot mint a run-scoped standing g // #given — a step-keyed record created by TRUSTED in-process code, bound to // the leg's exact suspension. The fix tightens the HTTP boundary only. const store = new InMemoryApprovalStore(); - const service = new ApprovalService({ store }); + const service = new ApprovalService({ store, executionFence: 'none' }); const suspendedAt = Date.parse('2026-07-09T00:00:00.000Z'); const { record } = await service.create( { diff --git a/packages/flowsafe/src/approval-api/principal-identity.ts b/packages/flowsafe/src/approval-api/principal-identity.ts new file mode 100644 index 0000000..5601e4c --- /dev/null +++ b/packages/flowsafe/src/approval-api/principal-identity.ts @@ -0,0 +1,78 @@ +// SPDX-License-Identifier: Apache-2.0 +// The execution principal's IDENTITY vocabulary — the kind list, the two +// predicates that police it, and the bound every principal-authored string +// obeys. Nothing else. +// +// WHY IT IS ITS OWN FILE. This module imports NOTHING, and that is the whole +// point of it rather than a happy accident. `principal.ts` imports `contract.ts` +// and carries the trusted-automation machinery; `start-idempotency.ts` declares +// six `DoStatusError` subclasses at module-eval time and needs exactly these +// three names to validate a stored row's owner. Reaching them through +// `principal.ts` puts a module-eval-time class hierarchy behind an import graph +// that is one edge away from cycling back — the same temporal-dead-zone hazard +// that forced `do-status-error.ts` out on its own. A leaf with no imports cannot +// participate in a cycle at all, so the hazard is closed by construction rather +// than by watching the graph. +// +// `principal.ts` re-exports every name here, so this split is invisible to +// existing importers and nothing outside this package can tell it happened. + +/** + * Mirrors breakwater's `PrincipalKind` by value, for the same reason + * contract.ts mirrors the request-context keys: flowsafe does not import + * breakwater at runtime. The cross-package contract test pins the equality. + */ +export const EXECUTION_PRINCIPAL_KINDS = [ + 'human', + 'service', + 'agent', + 'system', +] as const; + +export type ExecutionPrincipalKind = (typeof EXECUTION_PRINCIPAL_KINDS)[number]; + +/** Internal runtime guard for values crossing storage and request boundaries. */ +export function isExecutionPrincipalKind( + value: unknown, +): value is ExecutionPrincipalKind { + return ( + typeof value === 'string' && + (EXECUTION_PRINCIPAL_KINDS as readonly string[]).includes(value) + ); +} + +/** Upper bound on the free-text provenance fields, so audit rows stay bounded. */ +export const MAX_PURPOSE_LENGTH = 200; +export const MAX_PRINCIPAL_ID_LENGTH = 200; + +/** + * Bounded, non-empty, and free of control characters. + * + * Not an injection barrier for the wire — the principal travels through + * `JSON.stringify`, which escapes U+0000–U+001F. It matters because these + * fields do not stop at the wire: `id` becomes `requestedBy`/`decidedBy` in D1 + * and the actor on every audit row, and `purpose` rides into the SIEM export. + * The bounds keep an audit row bounded; the control-character refusal keeps + * those strings clean at the boundary rather than downstream. + * + * Exported for `principal.ts`'s own use only, and deliberately off the package + * barrel: it is one half of an internal validation rule, not public API. + */ +export function boundedText(value: unknown, max: number): value is string { + if (typeof value !== 'string' || value.trim() === '' || value.length > max) { + return false; + } + for (const character of value) { + const code = character.charCodeAt(0); + if (code <= 0x1f || code === 0x7f) return false; + } + return true; +} + +/** + * Canonical execution-principal identifier rule shared by every request and + * storage hydration boundary. Kept off the package barrel deliberately. + */ +export function isExecutionPrincipalId(value: unknown): value is string { + return boundedText(value, MAX_PRINCIPAL_ID_LENGTH); +} diff --git a/packages/flowsafe/src/approval-api/principal.test.ts b/packages/flowsafe/src/approval-api/principal.test.ts index c0d1324..d6af350 100644 --- a/packages/flowsafe/src/approval-api/principal.test.ts +++ b/packages/flowsafe/src/approval-api/principal.test.ts @@ -41,6 +41,9 @@ function harness(): { service: ApprovalService; events: ApprovalAuditEvent[] } { return { service: new ApprovalService({ store: backend.store(), + // In-memory store, no database to fence against: the opt-out is written down + // rather than defaulted — see ExecutionFenceWiring. + executionFence: 'none', audit: (event) => events.push(event), }), events, @@ -501,6 +504,7 @@ describe('automated provenance reaches subordinate audit events', () => { const events: ApprovalAuditEvent[] = []; const service = new ApprovalService({ store: backend.store(), + executionFence: 'none', audit: (event) => events.push(event), notify: () => { throw new Error('sink down'); @@ -536,6 +540,7 @@ describe('automated provenance reaches subordinate audit events', () => { const events: ApprovalAuditEvent[] = []; const service = new ApprovalService({ store: backend.store(), + executionFence: 'none', audit: (event) => events.push(event), stream: () => { throw new Error('hub down'); diff --git a/packages/flowsafe/src/approval-api/principal.ts b/packages/flowsafe/src/approval-api/principal.ts index 1343574..4b49eac 100644 --- a/packages/flowsafe/src/approval-api/principal.ts +++ b/packages/flowsafe/src/approval-api/principal.ts @@ -23,30 +23,29 @@ import { type ApprovalActor, type ApprovalRole, } from './contract.js'; +import { + boundedText, + type ExecutionPrincipalKind, + isExecutionPrincipalId, + isExecutionPrincipalKind, + MAX_PURPOSE_LENGTH, +} from './principal-identity.js'; /** - * Mirrors breakwater's `PrincipalKind` by value, for the same reason - * contract.ts mirrors the request-context keys: flowsafe does not import - * breakwater at runtime. The cross-package contract test pins the equality. + * The identity vocabulary lives on an import-free leaf (principal-identity.ts) + * so that modules which need only the kind list and the two predicates — the + * start-reservation store, which declares a class hierarchy at module-eval time + * — can take them without pulling this file's import graph behind them. + * + * Re-exported here rather than moved away, because THIS is the documented home + * of the execution principal and every existing importer names it. */ -export const EXECUTION_PRINCIPAL_KINDS = [ - 'human', - 'service', - 'agent', - 'system', -] as const; - -export type ExecutionPrincipalKind = (typeof EXECUTION_PRINCIPAL_KINDS)[number]; - -/** Internal runtime guard for values crossing storage and request boundaries. */ -export function isExecutionPrincipalKind( - value: unknown, -): value is ExecutionPrincipalKind { - return ( - typeof value === 'string' && - (EXECUTION_PRINCIPAL_KINDS as readonly string[]).includes(value) - ); -} +export { + EXECUTION_PRINCIPAL_KINDS, + type ExecutionPrincipalKind, + isExecutionPrincipalId, + isExecutionPrincipalKind, +} from './principal-identity.js'; /** Automated kinds — everything that is not a logged-in person. */ export const AUTOMATED_PRINCIPAL_KINDS: readonly ExecutionPrincipalKind[] = [ @@ -57,10 +56,6 @@ export const AUTOMATED_PRINCIPAL_KINDS: readonly ExecutionPrincipalKind[] = [ export type AutomatedPrincipalKind = Exclude; -/** Upper bound on the free-text provenance fields, so audit rows stay bounded. */ -const MAX_PURPOSE_LENGTH = 200; -const MAX_PRINCIPAL_ID_LENGTH = 200; - /** * `purpose` is REQUIRED on every automated kind, not optional as the roadmap * sketch had it. The failure being fixed is that fabricated operators "lose @@ -252,35 +247,6 @@ export function canonicalAutomatedPrincipal( : undefined; } -/** - * Bounded, non-empty, and free of control characters. - * - * Not an injection barrier for the wire — the principal travels through - * `JSON.stringify`, which escapes U+0000–U+001F. It matters because these - * fields do not stop at the wire: `id` becomes `requestedBy`/`decidedBy` in D1 - * and the actor on every audit row, and `purpose` rides into the SIEM export. - * The bounds keep an audit row bounded; the control-character refusal keeps - * those strings clean at the boundary rather than downstream. - */ -function boundedText(value: unknown, max: number): value is string { - if (typeof value !== 'string' || value.trim() === '' || value.length > max) { - return false; - } - for (const character of value) { - const code = character.charCodeAt(0); - if (code <= 0x1f || code === 0x7f) return false; - } - return true; -} - -/** - * Canonical execution-principal identifier rule shared by every request and - * storage hydration boundary. Kept off the package barrel deliberately. - */ -export function isExecutionPrincipalId(value: unknown): value is string { - return boundedText(value, MAX_PRINCIPAL_ID_LENGTH); -} - /** * Validate AND canonicalize in one pass: every field is read exactly once, and * the result is a fresh plain object built from THOSE reads. diff --git a/packages/flowsafe/src/approval-api/router.test.ts b/packages/flowsafe/src/approval-api/router.test.ts index 538b55f..de472e8 100644 --- a/packages/flowsafe/src/approval-api/router.test.ts +++ b/packages/flowsafe/src/approval-api/router.test.ts @@ -29,7 +29,10 @@ function makeHandler( return id && role ? { id, role: role as ApprovalRole } : undefined; }, storeFactory: backend, - buildService: (boundStore) => new ApprovalService({ store: boundStore }), + // In-memory store, no database to fence against: the opt-out is written down + // rather than defaulted — see ExecutionFenceWiring. + buildService: (boundStore) => + new ApprovalService({ store: boundStore, executionFence: 'none' }), }); const resolve = async (request: Request) => { const context = await resolveActor(request); diff --git a/packages/flowsafe/src/approval-api/router.ts b/packages/flowsafe/src/approval-api/router.ts index 623d4f5..9af8be4 100644 --- a/packages/flowsafe/src/approval-api/router.ts +++ b/packages/flowsafe/src/approval-api/router.ts @@ -15,6 +15,7 @@ // records that carry grants are minted in-process from an observed suspension, // never from a request body. +import { isExecutionFenceRefusal } from '../do-runner/execution-fence.js'; import { isPathSafeId } from '../do-runner/path-safe-id.js'; import { readBoundedBody } from '../http-body.js'; import { ActorResolutionError, type ActorResolver } from './actor-context.js'; @@ -126,6 +127,13 @@ function json(payload: unknown, status = 200): Response { } function errorResponse(error: unknown): Response { + // The deployment execution fence refusing, or failing to answer. 503 with + // its reason code, never the generic 500 below: a decision refused because + // this deployment is being migrated is retryable, and a reviewer's client + // must be able to tell that from a broken queue. + if (isExecutionFenceRefusal(error)) { + return json({ error: error.message, reason: error.reason }, error.status); + } if (error instanceof ApprovalPayloadTooLargeError) { return json({ error: error.message }, 413); } @@ -330,7 +338,7 @@ export function createApprovalRouter( } if (!isPathSafeId(runId)) { throw new InvalidApprovalInputError( - `runId '${runId}' is not path-safe — approvals bind to server-minted runs (INV-1)`, + `runId '${runId}' is not path-safe — approvals bind to server-minted runs`, ); } // The optional public filing route may only attach a request to a diff --git a/packages/flowsafe/src/approval-api/service.test.ts b/packages/flowsafe/src/approval-api/service.test.ts index 8f75435..36def4f 100644 --- a/packages/flowsafe/src/approval-api/service.test.ts +++ b/packages/flowsafe/src/approval-api/service.test.ts @@ -1,6 +1,14 @@ // SPDX-License-Identifier: Apache-2.0 import { describe, expect, it, vi } from 'vitest'; +import { openSqlite, sqliteUnitDatabase } from '../../test-support/sqlite.js'; +import { + type ExecutionFenceDatabase, + ExecutionFencedError, + type ExecutionFenceState, + ExecutionFenceStore, +} from '../do-runner/index.js'; + import type { ApprovalActor, ApprovalAuditEvent, @@ -66,6 +74,9 @@ function makeHarness(options: Partial = {}): Harness { let nowMs = T0; const service = new ApprovalService({ store, + // In-memory store, no database to fence against: the opt-out is written down + // rather than defaulted — see ExecutionFenceWiring. + executionFence: 'none', audit: (event) => events.push(event), now: () => new Date(nowMs), ...options, @@ -757,6 +768,7 @@ describe('ApprovalService.decide', () => { const events: ApprovalAuditEvent[] = []; const service = new ApprovalService({ store, + executionFence: 'none', audit: (event) => events.push(event), now: harness.now, }); @@ -861,9 +873,9 @@ describe('ApprovalService.sweepSLA', () => { it('escalates ALL breached requests across more than one page (>MAX)', async () => { // #given — more breached requests than a single page holds, so the sweep - // must cursor-page the (un-defaulted) system view instead of one SELECT; - // pre-D3 this bare list() was unbounded, post-D3 a naive single list would - // silently cap at MAX and leave the tail un-escalated + // must cursor-page the un-defaulted system view instead of one SELECT. + // Before the list gained its default bound it was unbounded; a naive single + // bounded list would silently cap at MAX and leave the tail un-escalated. const harness = makeHarness(); const total = MAX_APPROVAL_LIST_LIMIT + 1; for (let index = 0; index < total; index += 1) { @@ -989,6 +1001,7 @@ describe('ApprovalService.sweepSLA', () => { await seedPending(harness, { slaSeconds: 60 }); const bravoService = new ApprovalService({ store: harness.backend.store(), + executionFence: 'none', now: harness.now, }); await bravoService.create( @@ -2467,3 +2480,114 @@ describe('ApprovalService audit isolation', () => { expect(record.status).toBe('pending'); }); }); + +describe('ApprovalService.decide and the deployment execution fence', () => { + async function fenceAt( + state: ExecutionFenceState, + ): Promise { + const fence = new ExecutionFenceStore( + sqliteUnitDatabase(openSqlite()) as ExecutionFenceDatabase, + ); + await fence.seed(state); + return fence; + } + + it('still decides while draining', async () => { + // #given — a drain finishes outstanding work, and a decision is what an + // outstanding suspended run is waiting for. + const resumeRun = vi.fn(async () => undefined); + const harness = makeHarness({ + resumeRun, + executionFence: await fenceAt('draining'), + }); + const record = await seedPending(harness, { stepPath: ['approval'] }); + + // #then + const decided = await harness.service.decide( + record.id, + { decision: 'approve' }, + REVIEWER, + ); + expect(decided.record.status).toBe('approved'); + expect(resumeRun).toHaveBeenCalledTimes(1); + }); + + it('refuses a decision under migration-locked WITHOUT committing it', async () => { + // #given — decide() commits and THEN resumes, so a gate any later would + // leave the approval durably decided on a deployment that can never act on + // it, and the deployment taking over would inherit a decision with no + // resume behind it. + const resumeRun = vi.fn(async () => undefined); + const harness = makeHarness({ + resumeRun, + executionFence: await fenceAt('migration-locked'), + }); + const record = await seedPending(harness, { stepPath: ['approval'] }); + + // #when / #then + const refusal = await harness.service + .decide(record.id, { decision: 'approve' }, REVIEWER) + .catch((error: unknown) => error); + expect(refusal).toBeInstanceOf(ExecutionFencedError); + expect((refusal as ExecutionFencedError).reason).toEqual({ + code: 'EXECUTION_FENCED', + state: 'migration-locked', + }); + expect(resumeRun).not.toHaveBeenCalled(); + // #and — the record is untouched, so the migration carries a pending + // approval rather than a decided-but-unresumed one. + await expect(harness.store.get(record.id)).resolves.toMatchObject({ + status: 'pending', + }); + }); + + it('keeps list and read open while locked', async () => { + const harness = makeHarness({ + executionFence: await fenceAt('migration-locked'), + }); + const record = await seedPending(harness); + + await expect( + harness.service.get(record.id, REVIEWER), + ).resolves.toMatchObject({ id: record.id }); + await expect(harness.service.list({}, REVIEWER)).resolves.toEqual([ + expect.objectContaining({ id: record.id }), + ]); + }); + + it('decides only for the nominated proof run under proof-only', async () => { + // #given — a proof state bound to one run. + const fence = await fenceAt('migration-locked'); + await fence.transition({ + expected: 'migration-locked', + next: 'proof-only', + proofKey: 'proof-1', + }); + expect(await fence.recordProofRun('proof-1', 'acme_run-proof')).toBe(true); + const resumeRun = vi.fn(async () => undefined); + const harness = makeHarness({ resumeRun, executionFence: fence }); + const other = await seedPending(harness, { + runId: 'acme_run-other', + stepPath: ['approval'], + }); + const proof = await seedPending(harness, { + runId: 'acme_run-proof', + stepPath: ['approval'], + }); + + // #then — an approval that gates a different run is refused... + await expect( + harness.service.decide(other.id, { decision: 'approve' }, REVIEWER), + ).rejects.toBeInstanceOf(ExecutionFencedError); + + // #and — the proof run's gate is decided, which is what makes the proof + // able to reach a suspension and come back. + const decided = await harness.service.decide( + proof.id, + { decision: 'approve' }, + REVIEWER, + ); + expect(decided.record.status).toBe('approved'); + expect(resumeRun).toHaveBeenCalledTimes(1); + }); +}); diff --git a/packages/flowsafe/src/approval-api/service.ts b/packages/flowsafe/src/approval-api/service.ts index 4e67823..b5383c5 100644 --- a/packages/flowsafe/src/approval-api/service.ts +++ b/packages/flowsafe/src/approval-api/service.ts @@ -8,6 +8,12 @@ // (grants.ts) derives requestContext grants from approved records at // start/resume. Nothing here ever reads capability data from client input. +import { + admitsExistingRun, + ExecutionFencedError, + type ExecutionFenceWiring, + readExecutionFence, +} from '../do-runner/execution-fence.js'; import { isPathSafeId } from '../do-runner/path-safe-id.js'; import type { ApprovalActor, @@ -168,6 +174,24 @@ export interface ApprovalServiceOptions { * audited with `detail.selfDecision: true`. */ allowSelfDecision?: SelfDecisionPolicy; + /** + * The deployment execution fence, consulted before a DECISION commits, or + * `'none'` for a service with no database to fence against. + * + * DECIDE is gated wherever resume is, because decide() COMMITS and THEN + * resumes (see #resume): a lock that only stopped the resume would leave the + * approval durably decided on a deployment that can never act on it, and the + * deployment taking over would inherit a decision with no resume behind it. + * Reads, claims, and delegation stay open in every state — they move no run. + * + * REQUIRED, and `'none'` has to be WRITTEN. This service is the last leaf of + * the wiring to admit an omission, and it is the leaf where an omission costs + * the most: an unfenced service looks identical to a fenced one until the day + * an operator locks the deployment, and then it records decisions the + * deployment can never act on. Naming the opt-out turns that into a decision + * someone made rather than one they missed. See ExecutionFenceWiring. + */ + executionFence: ExecutionFenceWiring; /** Injectable clock (tests, deterministic SLA math). */ now?: () => Date; } @@ -195,6 +219,7 @@ export class ApprovalService { decision: ApprovalDecision, ) => Promise; readonly #allowSelfDecision?: SelfDecisionPolicy; + readonly #executionFence: ExecutionFenceWiring; readonly #now: () => Date; constructor(options: ApprovalServiceOptions) { @@ -205,6 +230,7 @@ export class ApprovalService { this.#defaultSlaSeconds = options.defaultSlaSeconds; this.#resumeRun = options.resumeRun; this.#allowSelfDecision = options.allowSelfDecision; + this.#executionFence = options.executionFence; this.#now = options.now ?? (() => new Date()); } @@ -424,6 +450,35 @@ export class ApprovalService { return updated; } + /** + * The execution fence, for the one approval operation that moves a run. + * + * Placed BEFORE the CAS commit, not around the resume: decide() writes the + * decision and then resumes, so a check any later would leave the approval + * durably decided with nothing behind it. + * + * `proof-only` costs one extra read, because the proof is a specific RUN and + * an approval id says nothing about which run it gates until the record is + * loaded. That read happens only in proof-only — an operational state that + * lasts minutes — so the steady-state cost stays exactly one fence read. + * + * The wiring is resolved through `readExecutionFence` rather than by testing + * the field here: a written `'none'` and a store both arrive at this one + * definition of what "no fence" does, so the opt-out cannot be a ternary this + * gate gets subtly wrong. + */ + async #assertDecidable(id: string): Promise { + const reading = await readExecutionFence(this.#executionFence); + if (reading.state === 'open' || reading.state === 'draining') return; + const runId = + reading.state === 'proof-only' + ? (await this.#store.get(id))?.runId + : undefined; + if (!admitsExistingRun(reading, runId)) { + throw new ExecutionFencedError(reading.state, 'approval decision'); + } + } + async decide( id: string, input: { decision: ApprovalDecision; comment?: string }, @@ -436,6 +491,7 @@ export class ApprovalService { `approval:${id}`, ); this.#assertDecisionInput(input); + await this.#assertDecidable(id); // Role-scoped SoD: an exempt decider (allowSelfDecision: true, or a role // named in { roles }) skips the pre-read entirely; everyone else keeps // today's read-then-CAS self-request denial. @@ -822,8 +878,8 @@ export class ApprovalService { async metrics(actor: ApprovalActor): Promise { this.#authorize(actor, APPROVAL_ROLES, 'approval.read', 'approval'); - // D3: computation moved into the store contract (SQL aggregate on D1, - // JS reduction on in-memory) instead of loading every record here. + // Compute in the store contract (SQL aggregate on D1, JS reduction on + // in-memory) instead of loading every record here. return this.#store.metrics(this.#now().getTime()); } @@ -1007,13 +1063,13 @@ export class ApprovalService { if (!isNonEmptyString(input.runId)) { throw new InvalidApprovalInputError('runId is required'); } - // Cheap INV-1 belt at the only write path: an approval binds to a + // A cheap run-id check at the only write path: an approval binds to a // server-minted run, and every context a runId keys (D1 row, DO name, // URL path) requires the path-safe charset. A record filed under a // malformed runId would be an orphan no resume can ever reach. if (!isPathSafeId(input.runId)) { throw new InvalidApprovalInputError( - `runId '${input.runId}' is not path-safe — approvals bind to server-minted runs (INV-1)`, + `runId '${input.runId}' is not path-safe — approvals bind to server-minted runs`, ); } if (!isNonEmptyString(input.title)) { @@ -1346,12 +1402,11 @@ export async function sweepSLA( const at = now(); const nowIso = at.toISOString(); const escalated: ApprovalRecord[] = []; - // D3: page the deployment's open set with an explicit cursor. Keyset paging - // is on (createdAt, id); - // escalating a record only drops it from the pending/claimed filter and never - // changes its cursor position, so every currently-open record is visited - // exactly once — no skips, no repeats — and every breach as of `at` still - // escalates regardless of where it sits in FIFO order. + // Page the deployment's open set with an explicit cursor. Keyset paging is on + // (createdAt, id); escalating a record only drops it from the pending/claimed + // filter and never changes its cursor position. Every currently-open record + // is visited exactly once — no skips, no repeats — and every breach as of + // `at` still escalates regardless of where it sits in FIFO order. let after: string | undefined; for (;;) { const page = await store.list({ diff --git a/packages/flowsafe/src/approval-api/types.ts b/packages/flowsafe/src/approval-api/types.ts index 68f32d0..ac845ef 100644 --- a/packages/flowsafe/src/approval-api/types.ts +++ b/packages/flowsafe/src/approval-api/types.ts @@ -56,6 +56,19 @@ export type ConnectorApprovalGrant = scope: 'run'; }); +/** + * The table the durable approval queue lives in. + * + * It sits on this leaf rather than beside the store that creates it because two + * layers now need the name and only one of them may import the store: the D1 + * store builds its DDL and every query from it, and the deployment drain + * inventory (do-runner/inventory.ts) counts the records still awaiting a + * decision. do-runner may reach approval-api only through this file and its + * siblings, so a name declared in `d1-store.ts` would have had to be restated + * there — and a restated table name is one a rename silently leaves behind. + */ +export const APPROVALS_TABLE = 'flowsafe_approvals'; + /** * Statuses that still await a decision. 'escalated' stays decidable — * escalation raises visibility, it does not close the request. diff --git a/packages/flowsafe/src/approval-ui/client-router.pipeline.test.ts b/packages/flowsafe/src/approval-ui/client-router.pipeline.test.ts index 6a65047..fbf33c5 100644 --- a/packages/flowsafe/src/approval-ui/client-router.pipeline.test.ts +++ b/packages/flowsafe/src/approval-ui/client-router.pipeline.test.ts @@ -1,6 +1,7 @@ // SPDX-License-Identifier: Apache-2.0 -// Full-pipeline D3 proof: the real ApprovalApiClient → createApprovalRouter → -// ApprovalService → InMemoryApprovalStore, driven with NO limit at >MAX scale. +// Full-pipeline bounded-list proof: the real ApprovalApiClient → +// createApprovalRouter → ApprovalService → in-memory store, run without a +// limit at >MAX scale. // Lives in approval-ui (not approval-api) because the "client" under test is // approval-ui's ApprovalApiClient; approval-ui → approval-api is the allowed // import direction. DOM-free (no hook, no JSX), so it runs in the main test @@ -41,7 +42,7 @@ function seedRecord( } describe('approval pipeline (client → router → service → store)', () => { - it('a bare reviewer-ordered list() past MAX_APPROVAL_LIST_LIMIT bounds the page yet still surfaces the freshest critical at the top (D3)', async () => { + it('a bare reviewer-ordered list() past MAX_APPROVAL_LIST_LIMIT bounds the page yet still surfaces the freshest critical at the top', async () => { // #given — a shared backend behind the router, seeded directly (fast) with // MAX normal requests plus one critical created LAST (newest createdAt, so // a FIFO-then-cap page would drop it at position MAX+1) @@ -64,7 +65,10 @@ describe('approval pipeline (client → router → service → store)', () => { return id && role ? { id, role: role as ApprovalRole } : undefined; }, storeFactory: backend, - buildService: (boundStore) => new ApprovalService({ store: boundStore }), + // In-memory store, no database to fence against: the opt-out is written down + // rather than defaulted — see ExecutionFenceWiring. + buildService: (boundStore) => + new ApprovalService({ store: boundStore, executionFence: 'none' }), }); const handle = createApprovalRouter({ resolve }); @@ -88,7 +92,7 @@ describe('approval pipeline (client → router → service → store)', () => { }, }); - // #when — no limit: exercises the D3 store-boundary default through the + // #when — no limit: exercises the store-boundary default through the // whole stack const page = await client.list({ status: [...OPEN_STATUSES], diff --git a/packages/flowsafe/src/background-tasks/host.test.ts b/packages/flowsafe/src/background-tasks/host.test.ts index cbd85cb..893f307 100644 --- a/packages/flowsafe/src/background-tasks/host.test.ts +++ b/packages/flowsafe/src/background-tasks/host.test.ts @@ -1,5 +1,5 @@ // SPDX-License-Identifier: Apache-2.0 -// Track B host wiring + the B-S2 recovery seam (R-002 pin: the PUBLIC async +// Background-task host wiring + the recovery seam (the PUBLIC async // init(pubsub) fires recoverStaleTasks internally — no private method is // called). Execution-mode recovery is proven over the serialized, deployment-bound // D1 domains: workers subscribe before init publishes the recovered dispatch, @@ -12,12 +12,43 @@ import { beforeEach, describe, expect, it, vi } from 'vitest'; import { z } from 'zod'; import { openSqlite, sqliteUnitDatabase } from '../../test-support/sqlite.js'; -import { createD1Storage, createHostPubSub, init } from '../do-runner/index.js'; +import { + createD1Storage, + createHostPubSub, + type ExecutionFenceDatabase, + ExecutionFencedError, + type ExecutionFenceState, + ExecutionFenceStore, + init, +} from '../do-runner/index.js'; import { backgroundTasksStore, createBackgroundTaskD1Domains, } from './d1-storage.js'; -import { BackgroundTaskHost } from './host.js'; +import { + BackgroundTaskHost, + type BackgroundTaskHostOptions, + backgroundTaskManagerForTests, + FENCE_RESUME_SCAN_PAGE, + MAX_FENCE_RESUME_SCANS, + MAX_FENCE_RESUMES_PER_PASS, +} from './host.js'; + +/** + * The host under test with the fence defaulted. `'none'` is the honest wiring + * for the InMemoryStore / unit-sqlite hosts these cases build — the fence cases + * at the bottom of this file pass a real store, and pass it through this same + * helper, so an explicit fence always wins. + */ +function newBackgroundTaskHost( + options: Omit & + Partial>, +): BackgroundTaskHost { + return new BackgroundTaskHost({ + ...options, + executionFence: options.executionFence ?? 'none', + }); +} function baseTask(overrides: Record) { const now = new Date(); @@ -45,7 +76,10 @@ async function seededD1(): Promise { const storage = createD1Storage({ binding: sqliteUnitDatabase(sqlite) as never, }); - const { createWorkflow, createStep, runtime } = init({ storage }); + const { createWorkflow, createStep, runtime } = init( + { storage }, + { startIdempotency: 'none', executionFence: 'none' }, + ); const step = createStep({ id: 'noop', inputSchema: z.object({}), @@ -90,44 +124,48 @@ describe('BackgroundTaskHost — wiring', () => { { cleanup: { failedTtlMs: Number.NaN } }, { cleanup: { cleanupIntervalMs: 0 } }, ])('rejects invalid manager configuration synchronously: %o', (manager) => { - expect( - () => - new BackgroundTaskHost({ - mastra: new Mastra({ storage: new InMemoryStore() }), - pubsub: createHostPubSub(), - executors: {}, - manager, - }), + expect(() => + newBackgroundTaskHost({ + mastra: new Mastra({ storage: new InMemoryStore() }), + pubsub: createHostPubSub(), + executors: {}, + manager, + }), ).toThrow(RangeError); }); it('accepts deliberate zero concurrency, retry, TTL, and throttle values', () => { - expect( - () => - new BackgroundTaskHost({ - mastra: new Mastra({ storage: new InMemoryStore() }), - pubsub: createHostPubSub(), - executors: {}, - manager: { - globalConcurrency: 0, - perAgentConcurrency: 0, - progressThrottleMs: 0, - defaultRetries: { - maxRetries: 0, - retryDelayMs: 0, - maxRetryDelayMs: 0, - backoffMultiplier: 0, - }, - cleanup: { completedTtlMs: 0, failedTtlMs: 0 }, + expect(() => + newBackgroundTaskHost({ + mastra: new Mastra({ storage: new InMemoryStore() }), + pubsub: createHostPubSub(), + executors: {}, + manager: { + globalConcurrency: 0, + perAgentConcurrency: 0, + progressThrottleMs: 0, + defaultRetries: { + maxRetries: 0, + retryDelayMs: 0, + maxRetryDelayMs: 0, + backoffMultiplier: 0, }, - }), + cleanup: { completedTtlMs: 0, failedTtlMs: 0 }, + }, + }), ).not.toThrow(); }); - it('boot() re-registers the static executors (survives DO eviction, DL-015)', async () => { + it('boot() re-registers the static executors (survives DO eviction)', async () => { // #given - const executor: ToolExecutor = { execute: async () => ({ done: true }) }; - const host = new BackgroundTaskHost({ + const executed: unknown[] = []; + const executor: ToolExecutor = { + execute: async (args) => { + executed.push(args); + return { done: true }; + }, + }; + const host = newBackgroundTaskHost({ mastra: new Mastra({ storage: new InMemoryStore() }), pubsub: createHostPubSub(), executors: { longResearch: executor }, @@ -136,13 +174,23 @@ describe('BackgroundTaskHost — wiring', () => { // #when await host.boot(); - // #then — resolvable by name (the path a recovered task's step takes) - expect(host.manager.getStaticExecutor('longResearch')).toBe(executor); + // #then — resolvable by name (the path a recovered task's step takes), and + // it DELEGATES to the registered executor. Identity is deliberately not + // asserted: what is registered is the fence-gated wrapper (host.ts + // `#gated`), which is the seam that stops a locked deployment executing a + // task body — including one a recovery re-drive resolved by name. + const registered = + backgroundTaskManagerForTests(host).getStaticExecutor('longResearch'); + expect(registered).toBeDefined(); + await expect(registered?.execute({ topic: 'ai' })).resolves.toEqual({ + done: true, + }); + expect(executed).toEqual([{ topic: 'ai' }]); }); it('boot() is idempotent — a second call resolves without re-init', async () => { // #given - const host = new BackgroundTaskHost({ + const host = newBackgroundTaskHost({ mastra: new Mastra({ storage: new InMemoryStore() }), pubsub: createHostPubSub(), executors: {}, @@ -160,7 +208,7 @@ describe('BackgroundTaskHost — wiring', () => { const mastra = new Mastra({ storage }); const store = await backgroundTasksStore(mastra); await store.createTask(baseTask({ id: 'stranded', maxRetries: 0 })); - const host = new BackgroundTaskHost({ + const host = newBackgroundTaskHost({ mastra, pubsub: createHostPubSub(), executors: {}, @@ -179,13 +227,13 @@ describe('BackgroundTaskHost — wiring', () => { // #given — a task left 'running' WITH retry budget, so recovery re-queues it // (the maxRetries>0 branch, distinct from the maxRetries=0 -> failed branch). // globalConcurrency:0 makes checkConcurrency refuse the re-dispatch, freezing - // the task at the pending hand-off so the transition is observable WITHOUT the - // R-B2-blocked execution re-dispatching it straight back to running. + // the task at the pending hand-off so the transition is observable WITHOUT + // the executor re-dispatching the blocked task straight back to running. const storage = new InMemoryStore(); const mastra = new Mastra({ storage }); const store = await backgroundTasksStore(mastra); await store.createTask(baseTask({ id: 'retryable', maxRetries: 3 })); - const host = new BackgroundTaskHost({ + const host = newBackgroundTaskHost({ mastra, pubsub: createHostPubSub(), executors: {}, @@ -216,7 +264,7 @@ describe('BackgroundTaskHost — wiring', () => { createdAt: old, }), ); - const host = new BackgroundTaskHost({ + const host = newBackgroundTaskHost({ mastra, pubsub: createHostPubSub(), executors: {}, @@ -235,7 +283,7 @@ describe('BackgroundTaskHost — execution lifecycle', () => { it('unwinds a partially failed worker start without initializing or shutting down the manager', async () => { // #given const { mastra, pubsub } = await executionHostDependencies(); - const host = new BackgroundTaskHost({ + const host = newBackgroundTaskHost({ mastra, pubsub, execution: true, @@ -250,8 +298,11 @@ describe('BackgroundTaskHost — execution lifecycle', () => { vi.spyOn(mastra, 'stopWorkers').mockImplementation(async () => { calls.push('stop-workers'); }); - const init = vi.spyOn(host.manager, 'init'); - const managerShutdown = vi.spyOn(host.manager, 'shutdown'); + const init = vi.spyOn(backgroundTaskManagerForTests(host), 'init'); + const managerShutdown = vi.spyOn( + backgroundTaskManagerForTests(host), + 'shutdown', + ); // #when / #then await expect(host.boot()).rejects.toBe(primary); @@ -267,7 +318,7 @@ describe('BackgroundTaskHost — execution lifecycle', () => { it('unwinds a failed manager init in reverse order and preserves the primary error', async () => { // #given const { mastra, pubsub } = await executionHostDependencies(); - const host = new BackgroundTaskHost({ + const host = newBackgroundTaskHost({ mastra, pubsub, execution: true, @@ -278,11 +329,16 @@ describe('BackgroundTaskHost — execution lifecycle', () => { vi.spyOn(mastra, 'startWorkers').mockImplementation(async () => { calls.push('start-workers'); }); - vi.spyOn(host.manager, 'init').mockImplementation(async () => { - calls.push('manager-init'); - throw primary; - }); - vi.spyOn(host.manager, 'shutdown').mockImplementation(async () => { + vi.spyOn(backgroundTaskManagerForTests(host), 'init').mockImplementation( + async () => { + calls.push('manager-init'); + throw primary; + }, + ); + vi.spyOn( + backgroundTaskManagerForTests(host), + 'shutdown', + ).mockImplementation(async () => { calls.push('manager-shutdown'); }); vi.spyOn(mastra, 'stopWorkers').mockImplementation(async () => { @@ -302,7 +358,7 @@ describe('BackgroundTaskHost — execution lifecycle', () => { it('aggregates boot cleanup failures after the primary error in cleanup order', async () => { // #given const { mastra, pubsub } = await executionHostDependencies(); - const host = new BackgroundTaskHost({ + const host = newBackgroundTaskHost({ mastra, pubsub, execution: true, @@ -315,11 +371,16 @@ describe('BackgroundTaskHost — execution lifecycle', () => { vi.spyOn(mastra, 'startWorkers').mockImplementation(async () => { calls.push('start-workers'); }); - vi.spyOn(host.manager, 'init').mockImplementation(async () => { - calls.push('manager-init'); - throw primary; - }); - vi.spyOn(host.manager, 'shutdown').mockImplementation(async () => { + vi.spyOn(backgroundTaskManagerForTests(host), 'init').mockImplementation( + async () => { + calls.push('manager-init'); + throw primary; + }, + ); + vi.spyOn( + backgroundTaskManagerForTests(host), + 'shutdown', + ).mockImplementation(async () => { calls.push('manager-shutdown'); throw managerCleanup; }); @@ -350,17 +411,20 @@ describe('BackgroundTaskHost — execution lifecycle', () => { it('shuts the manager down before stopping workers', async () => { // #given const { mastra, pubsub } = await executionHostDependencies(); - const host = new BackgroundTaskHost({ + const host = newBackgroundTaskHost({ mastra, pubsub, execution: true, executors: {}, }); vi.spyOn(mastra, 'startWorkers').mockResolvedValue(); - vi.spyOn(host.manager, 'init').mockResolvedValue(); + vi.spyOn(backgroundTaskManagerForTests(host), 'init').mockResolvedValue(); await host.boot(); const calls: string[] = []; - vi.spyOn(host.manager, 'shutdown').mockImplementation(async () => { + vi.spyOn( + backgroundTaskManagerForTests(host), + 'shutdown', + ).mockImplementation(async () => { calls.push('manager-shutdown'); }); vi.spyOn(mastra, 'stopWorkers').mockImplementation(async () => { @@ -378,7 +442,7 @@ describe('BackgroundTaskHost — execution lifecycle', () => { // #given const storage = new InMemoryStore(); const pubsub = createHostPubSub(); - const host = new BackgroundTaskHost({ + const host = newBackgroundTaskHost({ mastra: new Mastra({ storage, pubsub }), pubsub, executors: {}, @@ -387,7 +451,7 @@ describe('BackgroundTaskHost — execution lifecycle', () => { // #when — do not await shutdown before racing a new enqueue against it. const shutdown = host.shutdown(); - const enqueue = host.manager.enqueue({ + const enqueue = host.enqueue({ runId: 'acme_r1', toolName: 'late', toolCallId: 'call-late', @@ -407,8 +471,66 @@ describe('BackgroundTaskHost — execution lifecycle', () => { ); }); + it('retries a registration that failed on a transient storage fault', async () => { + // #given — a host whose first registration cannot reach storage. The read + // routes hang off this memo, so caching that answer would keep the whole + // host answering for a queue it can already reach again, until an eviction + // nobody can schedule. + const storage = new InMemoryStore(); + const mastra = new Mastra({ storage }); + const executor: ToolExecutor = { execute: async () => ({ done: true }) }; + let reads = 0; + vi.spyOn(mastra, 'getStorage').mockImplementation(() => { + reads += 1; + if (reads === 1) throw new Error('D1_ERROR: storage unavailable'); + return storage as never; + }); + const host = newBackgroundTaskHost({ + mastra, + pubsub: createHostPubSub(), + executors: { longResearch: executor }, + }); + + // #when — the first boot fails on that read. + await expect(host.boot()).rejects.toThrow(/storage unavailable/); + + // #then — the next boot registers for real rather than replaying the + // rejection: the executors a recovered task resolves by name are there. + await expect(host.boot()).resolves.toBeUndefined(); + expect( + backgroundTaskManagerForTests(host).getStaticExecutor('longResearch'), + ).toBeDefined(); + }); + + it('keeps a deterministic configuration failure memoized', async () => { + // #given — execution mode over a store that is not the serialized D1 + // workflows domain. Nothing about that answer can change without a new + // host, so re-validating it on every boot would only re-read storage to + // reach the same refusal. + const storage = new InMemoryStore(); + const mastra = new Mastra({ storage }); + const host = newBackgroundTaskHost({ + mastra, + pubsub: createHostPubSub(), + execution: true, + executors: {}, + }); + await expect(host.boot()).rejects.toThrow( + /requires DurableObjectWorkflowsStorageD1/, + ); + + // #when — a second boot, watched from here so only the RETRY's reads count. + const getStorage = vi.spyOn(mastra, 'getStorage'); + await expect(host.boot()).rejects.toThrow( + /requires DurableObjectWorkflowsStorageD1/, + ); + + // #then — the same refusal, served from the memo without touching storage. + expect(getStorage).not.toHaveBeenCalled(); + }); + it('treats shutdown as terminal even when boot was never started', async () => { - const host = new BackgroundTaskHost({ + const host = newBackgroundTaskHost({ mastra: new Mastra({ storage: new InMemoryStore() }), pubsub: createHostPubSub(), executors: {}, @@ -422,7 +544,7 @@ describe('BackgroundTaskHost — execution lifecycle', () => { it('waits for an in-flight boot before tearing its initialized components down', async () => { // #given const { mastra, pubsub } = await executionHostDependencies(); - const host = new BackgroundTaskHost({ + const host = newBackgroundTaskHost({ mastra, pubsub, execution: true, @@ -442,20 +564,27 @@ describe('BackgroundTaskHost — execution lifecycle', () => { signalStart(); await startReleased; }); - vi.spyOn(host.manager, 'init').mockImplementation(async () => { - calls.push('manager-init'); - }); - vi.spyOn(host.manager, 'shutdown').mockImplementation(async () => { + vi.spyOn(backgroundTaskManagerForTests(host), 'init').mockImplementation( + async () => { + calls.push('manager-init'); + }, + ); + vi.spyOn( + backgroundTaskManagerForTests(host), + 'shutdown', + ).mockImplementation(async () => { calls.push('manager-shutdown'); }); vi.spyOn(mastra, 'stopWorkers').mockImplementation(async () => { calls.push('stop-workers'); }); - // #when + // #when — the shutdown arrives once boot is already PAST admission and + // inside startWorkers. (A shutdown that lands before admission is the + // separate fence-read race below, where boot starts nothing at all.) const boot = host.boot(); - const shutdown = host.shutdown(); await startEntered; + const shutdown = host.shutdown(); expect(calls).toEqual(['start-workers']); releaseStart(); await Promise.all([boot, shutdown]); @@ -472,18 +601,21 @@ describe('BackgroundTaskHost — execution lifecycle', () => { it('still stops workers and preserves a lone manager shutdown failure', async () => { // #given const { mastra, pubsub } = await executionHostDependencies(); - const host = new BackgroundTaskHost({ + const host = newBackgroundTaskHost({ mastra, pubsub, execution: true, executors: {}, }); vi.spyOn(mastra, 'startWorkers').mockResolvedValue(); - vi.spyOn(host.manager, 'init').mockResolvedValue(); + vi.spyOn(backgroundTaskManagerForTests(host), 'init').mockResolvedValue(); await host.boot(); const primary = new Error('manager shutdown'); const calls: string[] = []; - vi.spyOn(host.manager, 'shutdown').mockImplementation(async () => { + vi.spyOn( + backgroundTaskManagerForTests(host), + 'shutdown', + ).mockImplementation(async () => { calls.push('manager-shutdown'); throw primary; }); @@ -499,18 +631,20 @@ describe('BackgroundTaskHost — execution lifecycle', () => { it('aggregates manager and worker shutdown failures in operation order', async () => { // #given const { mastra, pubsub } = await executionHostDependencies(); - const host = new BackgroundTaskHost({ + const host = newBackgroundTaskHost({ mastra, pubsub, execution: true, executors: {}, }); vi.spyOn(mastra, 'startWorkers').mockResolvedValue(); - vi.spyOn(host.manager, 'init').mockResolvedValue(); + vi.spyOn(backgroundTaskManagerForTests(host), 'init').mockResolvedValue(); await host.boot(); const primary = new Error('manager shutdown'); const workerCleanup = new Error('worker shutdown'); - vi.spyOn(host.manager, 'shutdown').mockRejectedValue(primary); + vi.spyOn(backgroundTaskManagerForTests(host), 'shutdown').mockRejectedValue( + primary, + ); vi.spyOn(mastra, 'stopWorkers').mockRejectedValue(workerCleanup); // #when @@ -526,7 +660,7 @@ describe('BackgroundTaskHost — execution lifecycle', () => { }); }); -describe('BackgroundTaskHost — on D1 (R-B1: persistence + recovery seam, no body execution)', () => { +describe('BackgroundTaskHost — D1 persistence and recovery without body execution', () => { let storage: MastraCompositeStore; beforeEach(async () => { @@ -536,7 +670,7 @@ describe('BackgroundTaskHost — on D1 (R-B1: persistence + recovery seam, no bo it('warns once at boot that dispatched bodies cannot execute on a non-concurrent store', async () => { // #given const warn = vi.spyOn(console, 'warn').mockImplementation(() => {}); - const host = new BackgroundTaskHost({ + const host = newBackgroundTaskHost({ mastra: new Mastra({ storage }), pubsub: createHostPubSub(), executors: {}, @@ -545,7 +679,7 @@ describe('BackgroundTaskHost — on D1 (R-B1: persistence + recovery seam, no bo // #when await host.boot(); - // #then — the R-B1 limitation is loud, not a stray async throw at dispatch + // #then — the D1 execution limitation is loud, not a stray dispatch throw expect(warn).toHaveBeenCalledWith( expect.stringContaining('concurrent updates'), ); @@ -558,7 +692,7 @@ describe('BackgroundTaskHost — on D1 (R-B1: persistence + recovery seam, no bo const mastra = new Mastra({ storage }); const store = await backgroundTasksStore(mastra); await store.createTask(baseTask({ id: 'stranded', maxRetries: 0 })); - const host = new BackgroundTaskHost({ + const host = newBackgroundTaskHost({ mastra, pubsub: createHostPubSub(), executors: {}, @@ -582,7 +716,7 @@ describe('BackgroundTaskHost — on D1 (R-B1: persistence + recovery seam, no bo // #when await store.createTask(baseTask({ id: 'persisted', status: 'suspended' })); - // #then — durable persistence works on D1 regardless of execution (R-B1) + // #then — durable persistence works on D1 regardless of execution const got = await store.getTask('persisted'); expect(got?.id).toBe('persisted'); expect(got?.status).toBe('suspended'); @@ -622,7 +756,7 @@ describe('BackgroundTaskHost — execution-mode recovery on D1', () => { const pubsub = createHostPubSub(); const mastra = new Mastra({ storage, pubsub }); const execute = vi.fn(async () => ({ recovered: true })); - const host = new BackgroundTaskHost({ + const host = newBackgroundTaskHost({ mastra, pubsub, execution: true, @@ -636,9 +770,9 @@ describe('BackgroundTaskHost — execution-mode recovery on D1', () => { booted = true; await vi.waitFor( async () => { - expect( - (await host.manager.getTask('retryable-execution'))?.status, - ).toBe('completed'); + expect((await host.getTask('retryable-execution'))?.status).toBe( + 'completed', + ); }, { timeout: 5_000, interval: 10 }, ); @@ -648,3 +782,649 @@ describe('BackgroundTaskHost — execution-mode recovery on D1', () => { } }); }); + +describe('BackgroundTaskHost and the deployment execution fence', () => { + async function fenceAt( + state: ExecutionFenceState, + ): Promise { + const fence = new ExecutionFenceStore( + sqliteUnitDatabase(openSqlite()) as ExecutionFenceDatabase, + ); + await fence.seed(state); + return fence; + } + + function hostWith( + executionFence: ExecutionFenceStore, + executor: ToolExecutor, + ): BackgroundTaskHost { + return newBackgroundTaskHost({ + mastra: new Mastra({ storage: new InMemoryStore() }), + pubsub: createHostPubSub(), + executors: { longResearch: executor }, + executionFence, + }); + } + + it('starts nothing when shutdown arrives during the fence read', async () => { + // #given — the window the fence read opens: `boot()` is suspended reading + // the fence, so it is past `boot()`'s own shutdown guard but has not + // admitted dispatching yet. Subscribing and claiming from here would hand + // work to a manager that is already being torn down — and `#doShutdown` + // waits on this very attempt, so it would be waiting for the claim it is + // trying to prevent. + const { mastra, pubsub } = await executionHostDependencies(); + let releaseFence: () => void = () => undefined; + const fenceRead = new Promise((resolve) => { + releaseFence = resolve; + }); + const open = await fenceAt('open'); + const slowFence = { + read: async () => { + await fenceRead; + return open.read(); + }, + } as unknown as ExecutionFenceStore; + const host = newBackgroundTaskHost({ + mastra, + pubsub, + execution: true, + executors: {}, + executionFence: slowFence, + }); + const calls: string[] = []; + vi.spyOn(mastra, 'startWorkers').mockImplementation(async () => { + calls.push('start-workers'); + }); + vi.spyOn(backgroundTaskManagerForTests(host), 'init').mockImplementation( + async () => { + calls.push('manager-init'); + }, + ); + vi.spyOn( + backgroundTaskManagerForTests(host), + 'shutdown', + ).mockImplementation(async () => { + calls.push('manager-shutdown'); + }); + vi.spyOn(mastra, 'stopWorkers').mockImplementation(async () => { + calls.push('stop-workers'); + }); + + // #when — shutdown arrives while the fence read is still outstanding. + const boot = host.boot(); + const shutdown = host.shutdown(); + expect(calls).toEqual([]); + releaseFence(); + await Promise.all([boot, shutdown]); + + // #then — the boot re-checks after the read and bails: nothing was started, + // so there is nothing to stop and nothing left running behind the teardown. + expect(calls).toEqual([]); + }); + + it('abandons the alarm duty when shutdown arrives during its boot', async () => { + // #given — the alarm's own boot, held inside the fence read. `onAlarm` + // sweeps and then runs core's TTL cleanup, and both would land on a manager + // that is already being torn down. + const { mastra, pubsub } = await executionHostDependencies(); + let releaseFence: () => void = () => undefined; + const fenceRead = new Promise((resolve) => { + releaseFence = resolve; + }); + const open = await fenceAt('open'); + const host = newBackgroundTaskHost({ + mastra, + pubsub, + executors: {}, + executionFence: { + read: async () => { + await fenceRead; + return open.read(); + }, + } as unknown as ExecutionFenceStore, + }); + const cleanup = vi + .spyOn(backgroundTaskManagerForTests(host), 'cleanup') + .mockResolvedValue(undefined as never); + + // #when — teardown is requested while the alarm's boot is still reading. + const alarm = host.onAlarm(); + const shutdown = host.shutdown(); + releaseFence(); + await Promise.all([alarm, shutdown]); + + // #then — the duty is dropped rather than run against a shut-down manager. + expect(cleanup).not.toHaveBeenCalled(); + }); + + it('starts the dispatcher ONCE when two boots race the fence read', async () => { + // #given — the request path and onAlarm() call boot() independently, so + // two attempts genuinely interleave. Both are held inside the fence read, + // which is the window `#dispatching` cannot cover: it is assigned only once + // that read RESUMES. + const { mastra, pubsub } = await executionHostDependencies(); + let releaseFence: () => void = () => undefined; + const fenceRead = new Promise((resolve) => { + releaseFence = resolve; + }); + const open = await fenceAt('open'); + const slowFence = { + read: async () => { + await fenceRead; + return open.read(); + }, + } as unknown as ExecutionFenceStore; + const host = newBackgroundTaskHost({ + mastra, + pubsub, + execution: true, + executors: {}, + executionFence: slowFence, + }); + const startWorkers = vi + .spyOn(mastra, 'startWorkers') + .mockImplementation(async () => undefined); + const init = vi + .spyOn(backgroundTaskManagerForTests(host), 'init') + .mockImplementation(async () => undefined); + + // #when — both boots pass the `#dispatching` guard before either read ends. + const first = host.boot(); + const second = host.boot(); + releaseFence(); + await Promise.all([first, second]); + + // #then — ONE dispatcher. A second manager.init subscribes handleDispatch a + // second time, and every dispatch is then handled twice: double claim, + // double body execution — with `#managerNeedsShutdown`/`#workersNeedStop` + // being booleans, teardown would stop only one of the two. + expect(init).toHaveBeenCalledTimes(1); + expect(startWorkers).toHaveBeenCalledTimes(1); + + // #and — a later boot still rides the settled memo rather than re-starting. + await host.boot(); + expect(init).toHaveBeenCalledTimes(1); + expect(startWorkers).toHaveBeenCalledTimes(1); + }); + + /** + * Seed `count` suspended rows and make `resume()` do what core's handler + * does to the row — leave the suspended set — so a sweep that re-reads it + * sees the truth rather than a stub that never moves. + */ + async function parkedHost( + tasks: ReadonlyArray<{ id: string; suspendedAt: Date; parked: boolean }>, + ) { + const binding = sqliteUnitDatabase(openSqlite()) as never; + const storage = createD1Storage({ + binding, + domains: createBackgroundTaskD1Domains({ binding }), + }); + await storage.init(); + const pubsub = createHostPubSub(); + const mastra = new Mastra({ storage, pubsub }); + const store = await backgroundTasksStore(mastra); + for (const task of tasks) { + await store.createTask( + baseTask({ + id: task.id, + status: 'suspended', + suspendedAt: task.suspendedAt, + suspendPayload: task.parked + ? { 'flowsafe.executionFenced': { state: 'migration-locked' } } + : { awaiting: 'webhook' }, + }), + ); + } + const host = newBackgroundTaskHost({ + mastra, + pubsub, + executors: {}, + executionFence: await fenceAt('open'), + }); + const resume = vi + .spyOn(backgroundTaskManagerForTests(host), 'resume') + .mockImplementation(async (taskId: string) => { + await store.updateTask(taskId, { status: 'running' }); + return (await store.getTask(taskId)) as never; + }); + const stillSuspended = async () => + (await store.listTasks({ status: 'suspended' })).tasks.map((t) => t.id); + return { host, resume, stillSuspended }; + } + + it('drains a parked cohort larger than the boot budget, without eviction', async () => { + // #given — 30 tasks parked by the backstop, more than one boot budget. + const parked = Array.from({ length: 30 }, (_, index) => ({ + id: `parked-${String(index).padStart(2, '0')}`, + suspendedAt: new Date(1_800_000_000_000 + index * 1_000), + parked: true, + })); + const { host, resume, stillSuspended } = await parkedHost(parked); + + try { + // #when — the boot after reopening. + await host.boot(); + + // #then — exactly one boot budget. The rest are NOT dropped: still + // suspended, still marked, still inspectable. + expect(resume).toHaveBeenCalledTimes(MAX_FENCE_RESUMES_PER_PASS); + expect(await stillSuspended()).toHaveLength( + 30 - MAX_FENCE_RESUMES_PER_PASS, + ); + + // #when — the alarm the host already arms for cleanup fires. THIS is the + // lane convergence rides: nothing waits behind it, so it carries the + // larger budget and keeps scanning until a scan resumes nothing. + await host.onAlarm(); + + // #then — drained. No eviction, no operator action, no fresh instance: + // the earlier shape swept only from #startDispatching, so every later + // boot returned the settled memo and these five stayed parked forever. + expect(resume).toHaveBeenCalledTimes(30); + expect(await stillSuspended()).toEqual([]); + } finally { + await host.shutdown(); + } + }); + + it('reaches a stale parked cohort sitting behind a full page of newer suspensions', async () => { + // #given — the cross-cycle case: five rows parked by an EARLIER lock, then + // more than a full scan page of newer tool-suspends (a tool awaiting a + // webhook is suspended on purpose and must stay that way). Scanning newest + // first, page 0 is all tool-suspends, so an unpaged sweep would never see + // the old cohort again. + const stale = Array.from({ length: 5 }, (_, index) => ({ + id: `stale-${index}`, + suspendedAt: new Date(1_700_000_000_000 + index * 1_000), + parked: true, + })); + const newer = Array.from({ length: 120 }, (_, index) => ({ + id: `tool-${String(index).padStart(3, '0')}`, + suspendedAt: new Date(1_900_000_000_000 + index * 1_000), + parked: false, + })); + const { host, resume, stillSuspended } = await parkedHost([ + ...stale, + ...newer, + ]); + + try { + // #when + await host.boot(); + + // #then — every stale row reached, and not one tool-suspended row + // touched. + expect(resume).toHaveBeenCalledTimes(5); + expect(resume.mock.calls.map((call) => call[0] as string).sort()).toEqual( + stale.map((task) => task.id), + ); + expect((await stillSuspended()).sort()).toEqual( + newer.map((task) => task.id).sort(), + ); + } finally { + await host.shutdown(); + } + }); + + it("reaches a cohort past one wake's scan bound on the NEXT alarm", async () => { + // #given — the stranding case paging alone cannot fix: a parked cohort + // sitting behind more newer tool-suspends than one wake can page through + // (MAX_FENCE_RESUME_SCANS * FENCE_RESUME_SCAN_PAGE rows). Every wake used + // to restart at page 0, so every wake re-read the same marker-free prefix + // and stopped in the same place — the cohort was unreachable forever. + // + // The set is served through a stubbed `listTasks` rather than seeded into + // D1: the point under test is which PAGES successive wakes ask for, and + // materializing thousands of rows to observe that would only make the test + // slower to lie in the same way. + const beyondOneWake = MAX_FENCE_RESUME_SCANS * FENCE_RESUME_SCAN_PAGE; + const remaining = [ + ...Array.from({ length: beyondOneWake }, (_, index) => + baseTask({ + id: `tool-${String(index).padStart(4, '0')}`, + status: 'suspended', + suspendedAt: new Date(1_900_000_000_000 + index), + suspendPayload: { awaiting: 'webhook' }, + }), + ), + ...Array.from({ length: 3 }, (_, index) => + baseTask({ + id: `parked-${index}`, + status: 'suspended', + suspendedAt: new Date(1_700_000_000_000 + index), + suspendPayload: { + 'flowsafe.executionFenced': { state: 'migration-locked' }, + }, + }), + ), + ]; + const host = newBackgroundTaskHost({ + mastra: new Mastra({ storage: new InMemoryStore() }), + pubsub: createHostPubSub(), + executors: {}, + executionFence: await fenceAt('open'), + }); + const manager = backgroundTaskManagerForTests(host); + const pagesRead: number[] = []; + vi.spyOn(manager, 'listTasks').mockImplementation(async (filter) => { + const page = filter?.page ?? 0; + pagesRead.push(page); + const from = page * FENCE_RESUME_SCAN_PAGE; + return { + tasks: remaining.slice(from, from + FENCE_RESUME_SCAN_PAGE), + } as never; + }); + const resume = vi + .spyOn(manager, 'resume') + .mockImplementation(async (taskId: string) => { + // What core's handler does to the row: it leaves the suspended set. + remaining.splice( + remaining.findIndex((task) => task.id === taskId), + 1, + ); + return undefined as never; + }); + + try { + // #when — the first wake spends its whole scan budget finding nothing. + await host.boot(); + + // #then — it reached the bound without a single resume, and stopped one + // page short of the cohort. + expect(resume).not.toHaveBeenCalled(); + expect(pagesRead).toEqual( + Array.from({ length: MAX_FENCE_RESUME_SCANS }, (_, page) => page), + ); + + // #when — the next alarm fires. Nothing changed on the deployment; the + // only thing carried across is where the last wake stopped reading. + pagesRead.length = 0; + await host.onAlarm(); + + // #then — it CONTINUES there rather than restarting, and the cohort that + // was unreachable is drained. + expect(pagesRead[0]).toBe(MAX_FENCE_RESUME_SCANS); + expect(resume).toHaveBeenCalledTimes(3); + expect(resume.mock.calls.map((call) => call[0] as string).sort()).toEqual( + ['parked-0', 'parked-1', 'parked-2'], + ); + } finally { + await host.shutdown(); + } + }); + + it('claims nothing while locked, then completes the queued work after reopen', async () => { + // #given — a real execution-mode host on D1 with work already queued: one + // PENDING row waiting for a dispatcher, and one row a previous instance + // left stranded in 'running' with the DEFAULT maxRetries of 0. + const binding = sqliteUnitDatabase(openSqlite()) as never; + const domains = () => createBackgroundTaskD1Domains({ binding }); + const seedStorage = createD1Storage({ binding, domains: domains() }); + await seedStorage.init(); + const seedStore = await backgroundTasksStore( + new Mastra({ storage: seedStorage }), + ); + await seedStore.createTask( + baseTask({ id: 'queued', status: 'pending', startedAt: undefined }), + ); + await seedStore.createTask(baseTask({ id: 'stranded', status: 'running' })); + + const storage = createD1Storage({ binding, domains: domains() }); + await storage.init(); + const pubsub = createHostPubSub(); + const executionFence = await fenceAt('migration-locked'); + const execute = vi.fn(async () => ({ ran: true })); + const host = newBackgroundTaskHost({ + mastra: new Mastra({ storage, pubsub }), + pubsub, + execution: true, + executors: { longResearch: { execute } }, + executionFence, + }); + + try { + // #when — the fenced boot (a request, or the DO alarm that woke it). + await host.boot(); + // Nothing is asynchronous about "did not happen", so give the dispatch + // topic a real chance to deliver before concluding it did not. + await new Promise((resolve) => setTimeout(resolve, 50)); + + // #then — NOTHING was claimed. The pending row is still pending, so it + // is still visible to a nonterminal census and still owned by the queue. + expect(await host.getTask('queued')).toMatchObject({ + status: 'pending', + }); + // #and — the stranded row was NOT settled. This is the destructive path: + // recoverStaleTasks marks every stranded maxRetries-0 row 'failed' + // outright, before any executor is consulted, so an executor-level gate + // could never have prevented it. + expect(await host.getTask('stranded')).toMatchObject({ + status: 'running', + }); + expect(execute).not.toHaveBeenCalled(); + + // #when — the migration finishes and the operator reopens the fence. No + // operator action beyond that: the next boot is a request or an alarm. + await executionFence.transition({ + expected: 'migration-locked', + next: 'open', + }); + await host.boot(); + + // #then — the queued work runs, exactly once. + await vi.waitFor( + async () => { + expect((await host.getTask('queued'))?.status).toBe('completed'); + }, + { timeout: 5_000, interval: 10 }, + ); + expect(execute).toHaveBeenCalledTimes(1); + } finally { + await host.shutdown(); + } + }); + + it('re-drives parked tasks in BOUNDED passes, newest first', async () => { + // #given — a long lock parked more tasks than one boot should re-drive. The + // sweep runs on the boot path every request and every alarm waits behind, + // so an unbounded serial resume would stall the first request after a + // reopen by the whole queue length. + const PER_PASS = 25; + const binding = sqliteUnitDatabase(openSqlite()) as never; + const domains = () => createBackgroundTaskD1Domains({ binding }); + const seedStorage = createD1Storage({ binding, domains: domains() }); + await seedStorage.init(); + const seedStore = await backgroundTasksStore( + new Mastra({ storage: seedStorage }), + ); + const parked = PER_PASS + 5; + for (let index = 0; index < parked; index += 1) { + await seedStore.createTask( + baseTask({ + id: `parked-${String(index).padStart(2, '0')}`, + status: 'suspended', + suspendedAt: new Date(1_000 + index), + suspendPayload: { + 'flowsafe.executionFenced': { state: 'migration-locked' }, + }, + }), + ); + } + // #and — one task its own TOOL suspended. It carries no fence marker, so it + // must survive every pass untouched. + await seedStore.createTask( + baseTask({ + id: 'tool-suspended', + status: 'suspended', + suspendedAt: new Date(9_999), + suspendPayload: { awaitingWebhook: true }, + }), + ); + + const storage = createD1Storage({ binding, domains: domains() }); + await storage.init(); + const pubsub = createHostPubSub(); + const host = newBackgroundTaskHost({ + mastra: new Mastra({ storage, pubsub }), + pubsub, + executors: { longResearch: { execute: async () => ({ ran: true }) } }, + executionFence: await fenceAt('open'), + }); + const resumed: string[] = []; + vi.spyOn(backgroundTaskManagerForTests(host), 'resume').mockImplementation( + async (taskId: string) => { + resumed.push(taskId); + return undefined as never; + }, + ); + + // #when + try { + await host.boot(); + } finally { + await host.shutdown(); + } + + // #then — exactly one pass' worth, and the NEWEST suspensions, which is the + // cohort a fence just parked. The remainder stays suspended and marked for + // the next boot or alarm to take. + expect(resumed).toHaveLength(PER_PASS); + expect(resumed).not.toContain('tool-suspended'); + expect(resumed[0]).toBe(`parked-${String(parked - 1).padStart(2, '0')}`); + }); + + it('dispatches normally while draining', async () => { + // #given — the queue IS the work a drain exists to finish, so a drain must + // keep dispatching or it can never complete. + const binding = sqliteUnitDatabase(openSqlite()) as never; + const domains = () => createBackgroundTaskD1Domains({ binding }); + const seedStorage = createD1Storage({ binding, domains: domains() }); + await seedStorage.init(); + const seedStore = await backgroundTasksStore( + new Mastra({ storage: seedStorage }), + ); + await seedStore.createTask( + baseTask({ id: 'draining', status: 'pending', startedAt: undefined }), + ); + + const storage = createD1Storage({ binding, domains: domains() }); + await storage.init(); + const pubsub = createHostPubSub(); + const host = newBackgroundTaskHost({ + mastra: new Mastra({ storage, pubsub }), + pubsub, + execution: true, + executors: { longResearch: { execute: async () => ({ ran: true }) } }, + executionFence: await fenceAt('draining'), + }); + + // #then + try { + await host.boot(); + await vi.waitFor( + async () => { + expect((await host.getTask('draining'))?.status).toBe('completed'); + }, + { timeout: 5_000, interval: 10 }, + ); + } finally { + await host.shutdown(); + } + }); + + it('parks rather than fails when the backstop catches an in-flight dispatch', async () => { + // #given — the narrow race the wrapper exists for: a subscriber started + // while the fence was open reaches a task body after it closed. Core has + // already written status:'running' by then, so the only question is how the + // body EXITS. + const executionFence = await fenceAt('migration-locked'); + const ran = vi.fn(async () => ({ done: true })); + const host = hostWith(executionFence, { execute: ran }); + await host.boot(); + const registered = + backgroundTaskManagerForTests(host).getStaticExecutor('longResearch'); + const suspend = vi.fn(async (_data?: unknown) => undefined); + + // #when + await expect(registered?.execute({}, { suspend })).resolves.toBeUndefined(); + + // #then — SUSPENDED, not failed. A throw becomes outcome:'retry', and core + // retries only while retryCount < maxRetries — the default is 0, so the + // first throw would settle the row 'failed' and the work would be gone. + expect(ran).not.toHaveBeenCalled(); + expect(suspend).toHaveBeenCalledTimes(1); + expect(suspend).toHaveBeenCalledWith({ + 'flowsafe.executionFenced': { state: 'migration-locked' }, + }); + }); + + it('refuses loudly when core supplies no suspend to park with', async () => { + // #given — `suspend` is optional on core's ToolExecutor contract. With no + // way to park, refusing beats running a task body on a deployment whose + // state is being copied. + const host = hostWith(await fenceAt('migration-locked'), { + execute: async () => ({ done: true }), + }); + await host.boot(); + + // #then + await expect( + backgroundTaskManagerForTests(host) + .getStaticExecutor('longResearch') + ?.execute({}), + ).rejects.toBeInstanceOf(ExecutionFencedError); + }); + + it('boots and serves read routes while locked', async () => { + // #given — boot() is deliberately NOT fence-gated: `#booted` memoizes it, + // so a refusal there would be cached for the life of the isolate and would + // take down the read routes every host serves after booting. + const host = hostWith(await fenceAt('migration-locked'), { + execute: async () => ({ done: true }), + }); + + // #then + await expect(host.boot()).resolves.toBeUndefined(); + await expect(host.listTasks({ runId: 'abc_r1' })).resolves.toEqual({ + tasks: [], + total: 0, + }); + }); + + it('accepts an enqueue while draining and refuses one once locked', async () => { + // #given — refusing an enqueue during a drain would fail exactly the runs + // that are draining, because the caller is a tool call inside one. + const executionFence = await fenceAt('draining'); + const host = hostWith(executionFence, { + execute: async () => ({ done: true }), + }); + await host.boot(); + const payload = { + toolName: 'longResearch', + toolCallId: 'call-1', + args: {}, + agentId: 'agent-1', + runId: 'abc_r1', + }; + + // #then + await expect( + host.enqueue(payload, { executor: { execute: async () => ({}) } }), + ).resolves.toMatchObject({ + task: expect.objectContaining({ id: expect.any(String) }), + }); + + // #when — locked + await executionFence.transition({ + expected: 'draining', + next: 'migration-locked', + }); + + // #then — a row written now is work the migration would have to carry. + await expect( + host.enqueue(payload, { executor: { execute: async () => ({}) } }), + ).rejects.toBeInstanceOf(ExecutionFencedError); + }); +}); diff --git a/packages/flowsafe/src/background-tasks/host.ts b/packages/flowsafe/src/background-tasks/host.ts index 45b9fc3..c4a769b 100644 --- a/packages/flowsafe/src/background-tasks/host.ts +++ b/packages/flowsafe/src/background-tasks/host.ts @@ -1,27 +1,34 @@ // SPDX-License-Identifier: Apache-2.0 -// Track B (M-003): host a Mastra BackgroundTaskManager on a Durable Object, and -// survive DO eviction BY CONSTRUCTION (DL-015). +// Host a Mastra BackgroundTaskManager on a Durable Object and survive eviction +// BY CONSTRUCTION. // -// THE RECOVERY SEAM (spike B-S2, pinned against @mastra/core 1.50.0 dist — -// validation finding R-002). `recoverStaleTasks()` and `handleResume` are -// PRIVATE and `getStorage()` is async, so the alarm CANNOT call -// `recoverStaleTasks()` directly. But it does not need to: the PUBLIC async -// `manager.init(pubsub)` fires `recoverStaleTasks()` internally (manager.d.ts:21 -// -> chunk .init -> `await this.recoverStaleTasks()`), guarded by its own -// initPromise so it runs once per manager INSTANCE. A DO evicted mid-task leaves -// its task row 'running'/'pending' in D1; when the DO is next instantiated (a -// FRESH manager), `boot()` re-registers the static tool executors, starts the -// workflow workers, and only then calls `init(pubsub)`. Init's recovery resets a +// THE RECOVERY SEAM, pinned against @mastra/core 1.50.0 dist. +// `recoverStaleTasks()` and `handleResume` are PRIVATE, and `getStorage()` is +// async, so the alarm CANNOT call `recoverStaleTasks()` directly. It does not +// need to: the PUBLIC async `manager.init(pubsub)` fires `recoverStaleTasks()` +// internally. Its manager.d.ts:21 -> chunk .init path awaits +// `this.recoverStaleTasks()`, guarded by its own initPromise so it runs once +// per manager INSTANCE. A DO evicted mid-task leaves its task row +// 'running'/'pending' in D1. When the DO is next instantiated with a FRESH +// manager, `boot()` re-registers the static tool executors, starts the workflow +// workers, and only then calls `init(pubsub)`. Init's recovery resets a // stranded 'running' task (maxRetries > 0) to 'pending' and re-dispatches it; -// starting workers first guarantees that the workflow event published by that -// dispatch has a subscriber. Its workflow step resolves the executor by tool -// name via the re-registered static registry (the cross-process path core ships -// `registerStaticExecutor` for). The DO ALARM is what WAKES an evicted DO so -// this happens without waiting for a request. No private method is ever called; -// the seam is `registerStaticExecutor` + `startWorkers()` + `init(pubsub)`, all -// public. +// starting workers first guarantees that the workflow event has a subscriber. +// Its workflow step resolves the executor by tool name via the re-registered +// static registry: the cross-process path core ships `registerStaticExecutor` +// for this. The DO ALARM WAKES an evicted DO so this happens without waiting +// for a request. No private method is ever called; the seam is +// `registerStaticExecutor` + `startWorkers()` + `init(pubsub)`, all public. // -// v1 policy (DL-005/P8): connectors are foreground-only, so approval-carrying +// THE FENCE SPLITS THAT SEAM IN TWO. `registerStaticExecutor` claims +// nothing and always runs; `startWorkers()` + `init(pubsub)` are what make this +// instance a dispatcher, and behind a closed deployment execution fence they +// are held back entirely — init's recovery would otherwise fail every stranded +// row outright and re-claim every pending one. The held-back phase is retried +// on every later boot (request or alarm), so reopening the fence is all it +// takes to resume. See #ensureDispatching. +// +// v1 policy: connectors are foreground-only, so approval-carrying // tools never enter this suspend/resume topology. Background suspend/resume // stays available for NON-gated tools (e.g. a long research tool awaiting an // external webhook); such a task's resume mints no capability. @@ -37,13 +44,25 @@ // snapshot before deleting the task row. import { + type BackgroundTask, BackgroundTaskManager, type BackgroundTaskManagerConfig, + type EnqueueResult, + type TaskContext, + type TaskFilter, + type TaskListResult, + type TaskPayload, type ToolExecutor, } from '@mastra/core/background-tasks'; import type { Mastra } from '@mastra/core/mastra'; - -import type { HostPubSub } from '../do-runner/index.js'; +import { EXECUTION_FENCE_SUSPEND_KEY } from '../do-runner/execution-fence.js'; +import { + admitsDrainableExecution, + ExecutionFencedError, + type ExecutionFenceWiring, + type HostPubSub, + readExecutionFence, +} from '../do-runner/index.js'; import { finiteNonnegativeNumber, nonnegativeSafeInteger, @@ -56,6 +75,8 @@ import { SERIALIZED_WORKFLOWS_D1, } from './d1-storage.js'; +export { EXECUTION_FENCE_SUSPEND_KEY } from '../do-runner/execution-fence.js'; + export interface BackgroundTaskHostOptions { /** * The DO's Mastra — must carry a storage adapter whose @@ -96,6 +117,29 @@ export interface BackgroundTaskHostOptions { * the persistence/recovery-only behavior. */ execution?: boolean; + /** + * The deployment execution fence, or `'none'` for a host with no database + * behind it. Task bodies are an execution family that runs BELOW + * RunnerRuntime, so the runtime's start/resume gate does not see them; this + * is where they are gated instead. + * + * The gate is at the DISPATCHER, read once per boot pass: behind a closed + * fence this instance never subscribes and never runs stale-task recovery, + * so nothing is claimed — queued rows stay pending and stranded rows stay + * stranded (#ensureDispatching explains why anything later than that is too + * late). The executor wrapper remains as a fail-closed backstop for the + * in-flight race, and parks rather than fails. + * + * REQUIRED: this is the ONLY gate task bodies pass, so an unfenced host runs + * them straight through a migration lock — and it looks wired, because every + * other surface of the same deployment reports the fence. See + * ExecutionFenceWiring. + * + * `draining` still dispatches and still accepts enqueues: that queue is the + * work a drain exists to finish, and refusing an enqueue would fail the very + * runs that are draining. + */ + executionFence: ExecutionFenceWiring; } function validateManagerConfig( @@ -178,22 +222,201 @@ function validateManagerConfig( } } +/** + * Was this row parked by the backstop, rather than suspended by its own tool? + * The distinction is the whole safety of the sweep: a task awaiting a webhook + * is suspended on purpose and must stay that way. + */ +function isFenceParked(task: BackgroundTask): boolean { + const payload = task.suspendPayload; + return ( + typeof payload === 'object' && + payload !== null && + EXECUTION_FENCE_SUSPEND_KEY in payload + ); +} + +/** + * How many fence-parked tasks ONE BOOT re-drives. + * + * Sized as a boot-path budget rather than a queue-drain rate: each resume is a + * publish onto the topic the manager just subscribed to, and the boot this + * blocks is what every request on the deployment is waiting behind. The + * remainder is not dropped — it stays `suspended`, marked, and inspectable + * through the read routes — and the ALARM lane below is what drains it, so a + * deployment reopening with a large parked queue starts moving on its first + * request without that request paying for the whole queue. + * + * Exported for the test that pins the boot/alarm budget boundary, and kept off + * `./index.js` — the only export map entry this directory has — so it stays + * package-internal rather than becoming a number a consumer can depend on. + */ +export const MAX_FENCE_RESUMES_PER_PASS = 25; + +/** + * How many one ALARM re-drives — the lane convergence actually rides on. + * + * Ten boot-budgets, because an alarm is a background duty and nothing is + * waiting behind it, unlike a boot. This is what makes the sweep terminate + * rather than merely make progress: the alarm recurs (the host arms it for + * `cleanup()` already), each wake drains up to this many, and the invocation + * itself keeps scanning until a scan finds nothing left to resume. + */ +const MAX_FENCE_RESUMES_PER_ALARM = 250; + +/** + * The hard bound on how many LIST queries one sweep invocation may issue. + * + * The sweep pages, and paging is what keeps an old parked cohort reachable + * once newer tool-suspends sit ahead of it in the scan order. Bounding the + * scans rather than only the resumes is what keeps a pathological mix — many + * thousands of tool-suspended rows and one stale parked cohort behind them — + * from turning a single wake into an unbounded read loop. + * + * A wake that hits this bound WITHOUT resuming anything has proved only that + * the first `MAX_FENCE_RESUME_SCANS * FENCE_RESUME_SCAN_PAGE` rows carry no + * marker, so it records where it stopped (`#sweepScanFloor`) and the next wake + * continues from there. Restarting every wake at page 0 instead would re-read + * the same prefix forever, and a cohort parked behind that many newer + * tool-suspends would never be reached at all. + * + * Exported alongside FENCE_RESUME_SCAN_PAGE for the same narrow reason + * MAX_FENCE_RESUMES_PER_PASS is: the test that proves a cohort past one wake's + * scan bound is still reached has to build a set that large, and a size + * hard-coded there would drift from these. Both stay off `./index.js` — they + * are tuning, not contract. + */ +export const MAX_FENCE_RESUME_SCANS = 20; + +/** + * How many `suspended` rows one pass READS to find that many. + * + * Larger than the resume budget on purpose. The fence marker lives in the + * suspend PAYLOAD, which no filter can express, so the pass has to read rows + * and sort them itself — and a page sized exactly to the resume budget would + * let a handful of tool-suspended tasks (awaiting a webhook, say) crowd the + * parked ones out of every pass and stall recovery indefinitely. Reading wider + * than it acts costs one query and buys headroom for that mix, while still + * bounding what a boot pulls out of D1. + */ +export const FENCE_RESUME_SCAN_PAGE = 100; + +/** + * A registration failure this host CHOSE, from wiring it can re-check without + * touching storage: the domains a host passed are the wrong ones. + * + * The distinction is what lets `boot()` memoize one kind of failure and retry + * the other. A configuration fault is deterministic — the next boot re-reads + * the same host wiring and fails identically — so caching it costs nothing and + * keeps the error stable. Everything else reaching #doRegister comes from a + * storage read (`getStorage()`, `getStore(...)`), and a transient D1 fault + * there must not lock this instance's read routes out until eviction. + * + * The split is deliberately conservative in one direction: a deterministic + * fault raised from INSIDE a storage read (a Mastra configured with no storage + * at all) is retried like a transient one, because from here the two are the + * same call. That costs one repeated read per boot and nothing else, whereas + * mistaking a transient fault for a permanent one costs the isolate's routes. + * + * Not exported: a caller cannot act on the difference, only this class can. + */ +class BackgroundTasksConfigurationError extends Error { + constructor(message: string) { + super(message); + this.name = 'BackgroundTasksConfigurationError'; + } +} + +/** + * Raw-manager access for THIS PACKAGE'S TESTS, and nothing else. + * + * A side table rather than a property ON the host, because a property is + * reachable whatever it is keyed by: an own symbol key is enumerable through + * `Object.getOwnPropertySymbols(host)[0]`, so "unreachable from the published + * package" was true of the NAME and false of the manager. Nothing outside this + * module can produce this WeakMap, so the accessor below — absent from + * `./index.js`, the only export map entry this directory has — is the only way + * in, and the claim is now literally true. + * + * It exists because the host's lifecycle contract IS its interleaving with + * core's manager (init before shutdown before stopWorkers, the enqueue guard + * flipping before the first await), and there is no way to observe that + * ordering from outside without a handle on the manager those calls land on. + * + * Production code must not use it: everything a host legitimately does goes + * through `enqueue` (fence-gated) or the three reads on the host itself. + */ +const hostManagers = new WeakMap(); + +/** @see hostManagers — tests only, and never re-exported. */ +export function backgroundTaskManagerForTests( + host: BackgroundTaskHost, +): BackgroundTaskManager { + const manager = hostManagers.get(host); + if (!manager) { + throw new Error('background-tasks: host has no manager registered'); + } + return manager; +} + /** * A BackgroundTaskManager bound to a hosting DO's Mastra + pubsub, with the * boot/alarm lifecycle that makes DO eviction survivable. The hosting DO owns * alarm arming (it needs `ctx.storage.setAlarm`); this class owns the manager - * wiring, the recovery-firing `boot()`, and the alarm `cleanup()` duty. The raw - * manager is reachable as `.manager` for the read-only routes to wrap — never - * expose it directly over HTTP. + * wiring, the recovery-firing `boot()`, and the alarm `cleanup()` duty. + * + * The manager itself is PRIVATE. It carries `enqueue`, `registerTaskContext`, + * `registerStaticExecutor`, `resume`, and `restart` — every one of which puts a + * task body on this deployment WITHOUT passing the fence, because the gate that + * stops a locked deployment executing is the `#gated` wrapper this host puts + * around an executor on the way in. A caller holding the manager could enqueue + * an unwrapped executor and defeat both. What this class forwards instead is + * the fence-gated `enqueue` and the three READS the host route adapter serves. */ export class BackgroundTaskHost { - readonly manager: BackgroundTaskManager; + readonly #manager: BackgroundTaskManager; readonly #mastra: Mastra; readonly #pubsub: HostPubSub; readonly #executors: Record; readonly #execution: boolean; + readonly #executionFence: ExecutionFenceWiring; #booted?: Promise; - #bootSettled = false; + /** Phase B (workers + init), memoized only once it has actually run. */ + #dispatching?: Promise; + /** + * The in-flight phase-B ATTEMPT, including the fence read that precedes + * admission — recorded synchronously by #ensureDispatching and cleared when + * it settles. `#dispatching` cannot serve this purpose: it is unset for the + * whole duration of the fence read, which is exactly the window two + * concurrent boots interleave in. + */ + #dispatchAttempt?: Promise; + /** + * The whole in-flight `boot()` — registration, the fence read, and phase B if + * the fence admitted it — recorded SYNCHRONOUSLY by `boot()`. + * + * Distinct from `#dispatching`, which is the memo of an ADMITTED phase and so + * is still unset while the fence read is in flight. Teardown waits on THIS: + * `shutdown()` called on a boot that has not reached admission yet would + * otherwise find nothing to wait for, return having stopped nothing, and + * leave behind the workers and the subscribed manager that boot went on to + * start. + */ + #bootAttempt?: Promise; + #bootAttemptSettled = false; + /** + * Where the NEXT sweep starts paging — the sweep's only state that outlives + * one invocation. + * + * Zero means "from the newest suspension", which is right whenever the last + * wake resumed something or reached the end of the set. It is non-zero only + * after a wake spent its whole scan budget finding no markers: that wake + * examined a prefix and proved it marker-free, so the next one continues + * past it instead of re-reading it. Instance state rather than storage + * because it is an optimisation, not a fact: an evicted DO simply starts + * from the top again, which is correct, only slower. + */ + #sweepScanFloor = 0; #managerNeedsShutdown = false; #workersNeedStop = false; #shutdownRequested = false; @@ -205,24 +428,39 @@ export class BackgroundTaskHost { this.#pubsub = options.pubsub; this.#executors = options.executors; this.#execution = options.execution ?? false; - this.manager = new BackgroundTaskManager({ + this.#executionFence = options.executionFence; + this.#manager = new BackgroundTaskManager({ enabled: true, ...options.manager, }); + hostManagers.set(this, this.#manager); // Must precede init(): the manager reads its Mastra for storage, the // internal background-task workflow registration, and id generation. - this.manager.__registerMastra(options.mastra); + this.#manager.__registerMastra(options.mastra); } /** - * Durable-Object boot wiring. Fail fast if the backgroundTasks storage domain is - * missing, re-register the static executors, start execution-mode workflow - * workers, THEN call `init(pubsub)` — whose internal `recoverStaleTasks()` - * re-drives any task the evicted instance left mid-flight. Executors and the - * workflow subscriber both go in BEFORE recovery publishes work. A failed - * startup unwinds attempted components in reverse order. Memoized per - * instance: `init` is itself idempotent (initPromise), and `boot()` from both - * `fetch()` and `alarm()` must not double-register. + * Durable-Object boot wiring, in two phases with different lifetimes. + * + * REGISTRATION (memoized for the life of the instance): validate the storage + * domains and re-register the static executors. Nothing here subscribes, + * claims, or recovers, so it is safe in every fence state — and it must be, + * because every host boots before routing and the read routes hang off this + * memo. + * + * DISPATCHING (fence-gated, re-attempted on every boot until it is admitted): + * `startWorkers()` plus `manager.init(pubsub)`. Those two are what make this + * instance CLAIM work — see #ensureDispatching for why they cannot run behind + * a closed fence. A refused attempt is deliberately not memoized, so the next + * `boot()` from a request or an alarm starts dispatching once the fence + * reopens, with no operator action. + * + * A registration that FAILED is memoized only when it can only fail again: + * the same non-memoized-refusal reasoning #ensureDispatching applies to the + * fence. A transient storage fault on the first request into a fresh isolate + * would otherwise be the permanent answer for the life of that isolate — the + * read routes hang off this memo, so the host would keep 500ing a queue it + * can now reach, until an eviction nobody can schedule. */ boot(): Promise { if (this.#shutdownRequested) { @@ -231,14 +469,36 @@ export class BackgroundTaskHost { ); } if (!this.#booted) { - this.#booted = this.#doBoot().finally(() => { - this.#bootSettled = true; - }); + const registration: Promise = this.#doRegister().catch( + (error: unknown) => { + // Identity-checked before clearing, the idiom `#ensureDispatching` + // and `shutdown()` use: a later registration must never be cleared + // by an earlier one's settlement. + if ( + !(error instanceof BackgroundTasksConfigurationError) && + this.#booted === registration + ) { + this.#booted = undefined; + } + throw error; + }, + ); + this.#booted = registration; } - return this.#booted; + const registered = this.#booted; + // Recorded before this method returns, so a `shutdown()` on the very next + // statement already has the whole attempt to wait on. + this.#bootAttemptSettled = false; + const attempt = registered + .then(() => this.#ensureDispatching()) + .finally(() => { + this.#bootAttemptSettled = true; + }); + this.#bootAttempt = attempt; + return attempt; } - async #doBoot(): Promise { + async #doRegister(): Promise { // Fail-fast with a clear message before any dispatch could surface core's // terser error deep in a lifecycle callback. const tasks = await backgroundTasksStore(this.#mastra); @@ -251,7 +511,7 @@ export class BackgroundTaskHost { SERIALIZED_WORKFLOWS_D1 ] !== true ) { - throw new Error( + throw new BackgroundTasksConfigurationError( 'background-tasks: execution requires DurableObjectWorkflowsStorageD1', ); } @@ -261,7 +521,7 @@ export class BackgroundTaskHost { DURABLE_OBJECT_BACKGROUND_TASKS_D1 ] !== true ) { - throw new Error( + throw new BackgroundTasksConfigurationError( 'background-tasks: execution requires DurableObjectBackgroundTasksStorageD1', ); } @@ -269,20 +529,103 @@ export class BackgroundTaskHost { await this.#warnIfBodiesCannotExecute(); } for (const [toolName, executor] of Object.entries(this.#executors)) { - this.manager.registerStaticExecutor(toolName, executor); + this.#manager.registerStaticExecutor(toolName, this.#gated(executor)); + } + } + + /** + * THE gate that matters: whether this instance becomes a DISPATCHER at all. + * + * `manager.init(pubsub)` is the only thing that subscribes `handleDispatch`, + * and `handleDispatch` is what writes `status: 'running'` — the CLAIM. It + * also runs `recoverStaleTasks()`, which on @mastra/core 1.53.0 does two + * destructive things behind a closed fence: + * + * 1. every stranded `running` row with `maxRetries === 0` (the DEFAULT) is + * marked `failed` outright, before any executor is consulted; and + * 2. every `pending` row is re-dispatched, i.e. claimed. + * + * Neither is reachable from inside an executor, which is why the gate lives + * here and not there. Skipping init leaves pending rows pending and stranded + * rows stranded — untouched, uncharged, and still visible to any nonterminal + * census — and the published dispatch event simply has no subscriber. One + * fence read per boot PASS, never memoized on refusal. + * + * `draining` still dispatches: the queue is exactly the work a drain exists + * to finish. + */ + #ensureDispatching(): Promise { + if (this.#dispatching) return this.#dispatching; + // DELIBERATELY NOT `async`, and the memo below is assigned before this + // method's first await can exist. + // + // `#dispatching` alone cannot collapse concurrent callers, because it is + // set only once the fence read RESUMES: two boots that both reach the + // guard above before either read completes would both go on to assign it, + // and both would run #startDispatching — `startWorkers()` twice and + // `manager.init(pubsub)` twice, which subscribes handleDispatch TWICE, so + // every dispatch is handled twice: double claim, double body execution. + // `#managerNeedsShutdown`/`#workersNeedStop` are booleans, so teardown + // would then stop one of the two. That is not hypothetical here: the + // request path and `onAlarm()` call `boot()` independently (a host's own + // memo collapses only the request path), so the two genuinely interleave. + // + // Recorded SYNCHRONOUSLY instead — the same shape `boot()` uses for + // `#bootAttempt`, for the same reason — and cleared when it settles, so a + // refusal still retries on the next boot. + const inFlight = this.#dispatchAttempt; + if (inFlight) return inFlight; + const attempt: Promise = this.#attemptDispatching().finally(() => { + // Identity-checked before clearing, the idiom `shutdown()` uses below: a + // later attempt must never be cleared by an earlier one's settlement. + if (this.#dispatchAttempt === attempt) this.#dispatchAttempt = undefined; + }); + this.#dispatchAttempt = attempt; + return attempt; + } + + async #attemptDispatching(): Promise { + const fence = await readExecutionFence(this.#executionFence); + // The fence read is an await, and `boot()`'s shutdown guard ran before it. + // A `shutdown()` that arrived in between has already decided this instance + // is going away; subscribing and claiming for the duration of an in-flight + // teardown would hand work to a manager that is about to stop — and + // `#doShutdown` waits on this very attempt, so it would wait for the claim + // it is trying to prevent. + if (this.#shutdownRequested) return; + if (!admitsDrainableExecution(fence)) { + console.warn( + JSON.stringify({ + type: 'background-tasks.dispatch-fenced', + state: fence.state, + reason: + 'workers and stale-task recovery are held back; queued tasks stay pending until the fence reopens', + }), + ); + return; } + this.#dispatching ??= this.#startDispatching().catch((error: unknown) => { + // Not memoized on failure, for the same reason a refusal is not: the + // next boot must be able to try again. + this.#dispatching = undefined; + throw error; + }); + return this.#dispatching; + } + + async #startDispatching(): Promise { try { if (this.#execution) { this.#workersNeedStop = true; await this.#mastra.startWorkers(); } this.#managerNeedsShutdown = true; - await this.manager.init(this.#pubsub); + await this.#manager.init(this.#pubsub); } catch (primary) { const cleanupErrors: unknown[] = []; if (this.#managerNeedsShutdown) { try { - await this.manager.shutdown(); + await this.#manager.shutdown(); this.#managerNeedsShutdown = false; } catch (error) { cleanupErrors.push(error); @@ -303,6 +646,229 @@ export class BackgroundTaskHost { { cause: primary }, ); } + // AFTER init, because resume() publishes onto the topic init subscribes. + // The BOOT budget: enough to start moving, small enough that the request + // that triggered this boot does not wait for the whole parked queue. + // onAlarm() is where the rest drains. + await this.#resumeFenceSuspendedTasks(MAX_FENCE_RESUMES_PER_PASS); + } + + /** + * Re-drive the tasks the executor backstop parked, once dispatching resumes. + * + * `recoverStaleTasks()` re-drives `running` and `pending` rows and knows + * nothing about `suspended` ones, so without this a task the backstop saved + * from destruction would be saved into a state nothing ever leaves. Scoped by + * the backstop's own marker: a task suspended by its TOOL (awaiting a + * webhook, say) is a different thing entirely and must stay suspended. + * + * BUDGETED, not one-shot. It runs on two lanes with two budgets: `boot()` + * spends MAX_FENCE_RESUMES_PER_PASS so the first request after a reopen is + * not stalled behind the whole parked queue, and `onAlarm()` spends + * MAX_FENCE_RESUMES_PER_ALARM on a lane nothing is waiting behind. The alarm + * lane is what makes this CONVERGE: one invocation keeps scanning until a + * scan resumes nothing, and the alarm recurs, so a queue larger than one + * alarm's budget still drains over the next few wakes with no eviction and no + * operator action. (An earlier shape swept only from #startDispatching, which + * runs once — every later boot returned the settled memo and never swept + * again, so anything past the first cap stayed parked indefinitely.) + * + * PAGES, because the scan order alone cannot reach a stale cohort. Newest + * suspensions first is right for the common case — the parked cohort is the + * one that just parked — but leftovers from an earlier lock carry an OLD + * `suspendedAt`, so after a lock/reopen/lock cycle plus a page-worth of newer + * tool-suspends, page 0 would never contain them again. So: a page that + * resumed nothing and came back FULL means the markers are deeper, and the + * sweep advances; a page that resumed something restarts at the top, because + * resuming removes rows from the set being paged and any fixed offset would + * then skip rows. `#resumedIds` makes that restart cheap and terminating — + * each pass strictly grows it. + * + * And the paging SURVIVES the wake, because within one wake it is bounded + * (MAX_FENCE_RESUME_SCANS). A cohort sitting behind more newer tool-suspends + * than one wake can page through would otherwise be stranded outright: every + * wake would re-read the same marker-free prefix and stop in the same place. + * So a wake that spends its whole scan budget finding nothing leaves + * `#sweepScanFloor` where it stopped and the next one resumes there, while + * any wake that resumes something — or reaches the end of the set — puts it + * back to zero, since both mean the prefix is worth re-reading. A floor left + * past the end of a shrunken set reads a short page and resets itself. + * + * The cost of that is bounded and the right way round: while the floor is + * deep, a cohort parked FRESH at the top waits the few wakes it takes to page + * to the end of the set (which resets the floor), instead of the old shape's + * "waits forever". Delaying the reachable cohort by a wake or two is the + * cheaper error than stranding the unreachable one permanently. + * + * `perPage`/`orderBy` are ADAPTER-DEPENDENT: honoured by + * @mastra/cloudflare-d1's SQL builder and by this package's D1 domain, but an + * adapter that ignores them returns the whole list — which is why the loop + * counts its own resumes rather than trusting the page size to bound them. + * + * Answers how many it resumed, so the caller can log a lane that is making + * progress distinctly from one that has drained. + */ + async #resumeFenceSuspendedTasks(budget: number): Promise { + // Resuming is asynchronous at the storage layer (resume() publishes; the + // handler is what writes 'running'), so a row can still read `suspended` on + // the next scan. Without this the sweep could re-resume the same rows — + // wasted publishes at best, and core throws outright once the row HAS + // moved, which would abort the whole invocation. + const resumedIds = new Set(); + let page = this.#sweepScanFloor; + // The floor the NEXT wake starts from. Any wake that resumed something has + // changed the set it was paging, so its prefix is worth re-reading and the + // floor goes back to zero whatever this is called with. + const finish = (nextFloor: number): number => { + this.#sweepScanFloor = resumedIds.size > 0 ? 0 : nextFloor; + return resumedIds.size; + }; + for (let scan = 0; scan < MAX_FENCE_RESUME_SCANS; scan += 1) { + let tasks: readonly BackgroundTask[]; + try { + ({ tasks } = await this.#manager.listTasks({ + status: 'suspended', + orderBy: 'suspendedAt', + orderDirection: 'desc', + page, + perPage: FENCE_RESUME_SCAN_PAGE, + })); + } catch (error) { + this.#logFenceResumeFailure(error); + // This page proved nothing, so the next wake retries it rather than + // stepping over rows it never read. + return finish(page); + } + let resumedThisScan = 0; + for (const task of tasks) { + if (resumedIds.size >= budget) return finish(0); + if (resumedIds.has(task.id) || !isFenceParked(task)) continue; + try { + await this.#manager.resume(task.id); + } catch (error) { + // Expected at capacity: core refuses a resume that would exceed the + // concurrency limit. Stopping is the honest response — the rows stay + // parked and the next wake retries once slots free — and it keeps a + // saturated deployment from burning the scan budget on refusals. + this.#logFenceResumeFailure(error); + return finish(page); + } + resumedIds.add(task.id); + resumedThisScan += 1; + } + if (resumedThisScan > 0) { + page = 0; + continue; + } + // Nothing here to resume. A SHORT page is the end of the set, so there is + // nothing deeper and this sweep has drained. A full one means the markers + // may simply be further down. + if (tasks.length < FENCE_RESUME_SCAN_PAGE) return finish(0); + page += 1; + } + // Out of scans with nothing found: `page` is the first row range this wake + // never read, and the next one starts there. + return finish(page); + } + + #logFenceResumeFailure(error: unknown): void { + // Best effort: a failure here leaves the rows suspended and inspectable, + // which is where they already were. Never fails the boot that was about to + // start serving, nor the alarm duty that follows. + console.error( + JSON.stringify({ + type: 'background-tasks.fence-resume-error', + reason: error instanceof Error ? error.message : String(error), + }), + ); + } + + /** + * The fail-closed BACKSTOP, not the gate. #ensureDispatching stops this + * instance claiming work behind a closed fence; this only covers the narrow + * race where a subscriber started while the fence was open consumes a + * dispatch event published just before it closed. By then core has ALREADY + * written `status: 'running'`, so the claim cannot be undone from here — only + * exited well. + * + * SUSPEND, never throw. A thrown executor becomes `outcome: 'retry'`, and + * core retries only while `retryCount < maxRetries` — the default maxRetries + * is 0, so the very first throw settles the row `failed`: work destroyed, and + * silently absent from any nonterminal census. `suspend()` instead persists + * `status: 'suspended'` with a payload naming the fence, which is nonterminal, + * inspectable, and re-driven by #resumeFenceSuspendedTasks after reopen. + * Throwing survives only as the last resort for a core that supplied no + * `suspend` (it is optional on the interface): refusing loudly beats running + * a task body on a deployment whose state is being copied. + */ + #gated(executor: ToolExecutor): ToolExecutor { + return { + execute: async (args, executeOptions) => { + const fence = await readExecutionFence(this.#executionFence); + if (!admitsDrainableExecution(fence)) { + const suspend = executeOptions?.suspend; + if (!suspend) { + throw new ExecutionFencedError(fence.state, 'background task'); + } + await suspend({ + [EXECUTION_FENCE_SUSPEND_KEY]: { state: fence.state }, + }); + // Discarded on the suspend path — see core's ToolExecutor contract. + return undefined; + } + return executor.execute(args, executeOptions); + }, + }; + } + + /** + * Enqueue a background task through the fence. + * + * A drain deliberately still ACCEPTS enqueues: the caller is an agent's tool + * call inside a run the drain is trying to finish, and refusing would fail + * exactly the runs that are draining. Locked and proof-only refuse, because + * a row written then is work the migration would have to carry. + * + * The per-task executor is wrapped with the same gate as the static ones, so + * a task enqueued while open cannot execute its body after a transition. + */ + async enqueue( + payload: TaskPayload, + context?: TaskContext, + ): Promise { + const fence = await readExecutionFence(this.#executionFence); + if (!admitsDrainableExecution(fence)) { + throw new ExecutionFencedError(fence.state, 'background task enqueue'); + } + return this.#manager.enqueue( + payload, + context + ? { ...context, executor: this.#gated(context.executor) } + : undefined, + ); + } + + // The read surface, and only the read surface. These three are what + // `createBackgroundTaskRoutes` serves and what a host inspecting its own + // queue needs; they claim nothing, so they stay open in every fence state + // (the semantics matrix keeps reads answering through a lock — an operator + // proving a deployment is drained has to be able to look). + + /** One task by id, or null. */ + getTask(taskId: string): Promise { + return this.#manager.getTask(taskId); + } + + /** Tasks matching a filter. The route adapter re-checks scope per row. */ + listTasks(filter?: TaskFilter): Promise { + return this.#manager.listTasks(filter); + } + + /** Lifecycle-event stream, for the route adapter's SSE response. */ + stream( + options?: Parameters[0], + ): ReadableStream> { + return this.#manager.stream(options); } async #warnIfBodiesCannotExecute(): Promise { @@ -328,7 +894,23 @@ export class BackgroundTaskHost { */ async onAlarm(): Promise { await this.boot(); - await this.manager.cleanup(); + // Teardown wins over the duty. A `shutdown()` that arrived while boot() was + // in flight got past its guard, so without this the alarm would sweep + // mid-teardown — publishing resumes onto a manager that is stopping — and + // then call cleanup() on a manager that has already shut down. + if (this.#shutdownRequested) return; + // The convergence lane. boot() has resolved, so a set `#dispatching` means + // init actually landed and resume() has a subscriber to publish to. Nothing + // is waiting behind an alarm, so this lane carries the larger budget and is + // what drains a parked queue the boot budget only dented — including on the + // very wake that admitted dispatching, since the alarm recurs regardless. + // + // Convergence therefore rides the same alarm the host already arms for the + // cleanup below; a host that arms none gets neither duty. + if (this.#dispatching) { + await this.#resumeFenceSuspendedTasks(MAX_FENCE_RESUMES_PER_ALARM); + } + await this.#manager.cleanup(); } /** @@ -337,18 +919,29 @@ export class BackgroundTaskHost { * teardown is still attempted if manager teardown fails. */ async #doShutdown(): Promise { - if (this.#booted && !this.#bootSettled) { + // ONE wait, on the whole boot attempt: registration, the fence read, and + // phase B all hang off it, so nothing a racing boot is about to start can + // slip past this point and outlive the shutdown. Waiting on `#dispatching` + // instead would miss exactly the window the fence read opens — it is unset + // until admission, so a shutdown that arrived during the read would stop + // nothing and return. + // + // Awaited ONLY while still in flight: manager.shutdown() below flips its + // enqueue guard before its first await, and that guarantee survives only if + // nothing suspends this function before it is invoked. + if (this.#bootAttempt && !this.#bootAttemptSettled) { try { - await this.#booted; + await this.#bootAttempt; } catch { - // #doBoot already unwound every component it managed to stop. Any - // component whose cleanup failed remains flagged for the retry below. + // #doRegister only validates and registers, and #startDispatching + // already unwound every component it managed to stop. Any component + // whose cleanup failed remains flagged for the retry below. } } const errors: unknown[] = []; if (this.#managerNeedsShutdown) { try { - await this.manager.shutdown(); + await this.#manager.shutdown(); this.#managerNeedsShutdown = false; } catch (error) { errors.push(error); diff --git a/packages/flowsafe/src/background-tasks/index.ts b/packages/flowsafe/src/background-tasks/index.ts index 8c8431c..333fa71 100644 --- a/packages/flowsafe/src/background-tasks/index.ts +++ b/packages/flowsafe/src/background-tasks/index.ts @@ -1,9 +1,8 @@ // SPDX-License-Identifier: Apache-2.0 -// Track B (M-003) — background tasks. Subpath-only (`@proofoftech/flowsafe/ -// background-tasks`), like agent-runner: host-side wiring a consumer opts into, -// not part of the root barrel. The completedAt TTL cleanup lives in do-runner -// (coupled there to the schema guard) and is -// re-exported here for a single import surface. +// Background tasks. Subpath-only (`@proofoftech/flowsafe/background-tasks`), +// like agent-runner: host-side wiring a consumer opts into, not part of the +// root barrel. The completedAt TTL cleanup lives in do-runner, coupled there to +// the schema guard, and is re-exported here for a single import surface. export type { CreateBackgroundTaskD1DomainsOptions, @@ -21,8 +20,15 @@ export { SERIALIZED_WORKFLOWS_D1, } from './d1-storage.js'; export type { BackgroundTaskHostOptions } from './host.js'; -export { BackgroundTaskHost } from './host.js'; +// EXECUTION_FENCE_SUSPEND_KEY is the marker a host reads to tell a +// fence-parked row from a tool-suspended one — the one thing about the +// backstop's parking a consumer legitimately needs. The resume budgets and +// scan bounds beside it stay package-internal (they are tuning, not contract), +// and `backgroundTaskManagerForTests` stays internal for a stronger reason: it +// hands out the ungated manager. +export { BackgroundTaskHost, EXECUTION_FENCE_SUSPEND_KEY } from './host.js'; export type { + BackgroundTaskReads, BackgroundTaskRouter, BackgroundTaskRoutesOptions, } from './routes.js'; diff --git a/packages/flowsafe/src/background-tasks/routes.ts b/packages/flowsafe/src/background-tasks/routes.ts index f2725ad..07ec693 100644 --- a/packages/flowsafe/src/background-tasks/routes.ts +++ b/packages/flowsafe/src/background-tasks/routes.ts @@ -1,5 +1,5 @@ // SPDX-License-Identifier: Apache-2.0 -// Track B (M-003): the read-only HTTP surface over background tasks (DL-014). +// The read-only HTTP surface over background tasks. // Core's `listTasks` / `getTask` / `stream` are not run-bound, so this router: // - requires a path-safe `runId` or `threadId` filter on list and stream; // - loads a task for `getTask` and 404s when missing; @@ -7,7 +7,7 @@ // Mutating the queue // (dispatch/cancel/resume) is NOT here: v1 keeps background dispatch server-side // (an agent's tool call), and a suspended non-gated task's resume, if ever -// exposed, is a separate role-gated route that mints no capability (P8). +// exposed, is a separate role-gated route that mints no capability. import type { BackgroundTaskManager, @@ -17,9 +17,27 @@ import type { import { isPathSafeId } from '../do-runner/index.js'; import { safeDecodeSegment } from '../host-kit/route-path.js'; +/** + * The three reads this router makes, as a type of their own. + * + * Narrower than `BackgroundTaskManager` on purpose. The manager also carries + * `enqueue`, `registerStaticExecutor`, `registerTaskContext`, `resume`, and + * `restart` — each of which puts a task body on the deployment without passing + * the execution fence — so a route handler holding one is a single property + * access away from the thing the fence exists to stop. Typing the option as the + * reads instead lets `BackgroundTaskHost` hand over its own fence-preserving + * forwarding surface, while a plain manager still satisfies it structurally for + * a host that has no fence to preserve. + */ +export interface BackgroundTaskReads { + getTask: BackgroundTaskManager['getTask']; + listTasks: BackgroundTaskManager['listTasks']; + stream: BackgroundTaskManager['stream']; +} + export interface BackgroundTaskRoutesOptions { - /** The manager to read through — WRAPPED, never exposed over the wire. */ - manager: BackgroundTaskManager; + /** The read surface to serve — WRAPPED, never exposed over the wire. */ + manager: BackgroundTaskReads; /** Host-owned authorization for the run/thread scope of every returned row. */ authorize(scope: { runId?: string; threadId?: string }): Promise; /** Route prefix. Default '/background-tasks'. */ @@ -136,7 +154,7 @@ export function createBackgroundTaskRoutes( return json({ error: 'not found' }, 404); } const result = await manager.listTasks(resolved.filter); - // Per-row parity with the stream guard (DL-014): re-check every returned + // Per-row parity with the stream guard: re-check every returned // row against the requested scope, so a future regression in core's // listTasks filter cannot leak a foreign or out-of-scope task. `total` is // recomputed from the surviving rows — never report a count that includes @@ -161,7 +179,7 @@ export function createBackgroundTaskRoutes( * disconnect closes the upstream subscription. */ function streamResponse( - manager: BackgroundTaskManager, + manager: BackgroundTaskReads, scoped: { filter: TaskFilter; scopeValue: string }, abortSignal: AbortSignal, ): Response { @@ -172,7 +190,7 @@ function streamResponse( new TransformStream, Uint8Array>({ transform(chunk, controller) { // Same per-row scope guard the list route applies — one predicate so the - // two surfaces never drift (DL-014, defense-in-depth over core's filter). + // two surfaces never drift (defense-in-depth over core's filter). const payload = typeof chunk.payload === 'object' && chunk.payload !== null && diff --git a/packages/flowsafe/src/cloudflare-fidelity.workerd.test.ts b/packages/flowsafe/src/cloudflare-fidelity.workerd.test.ts index b59d9fb..1b17de6 100644 --- a/packages/flowsafe/src/cloudflare-fidelity.workerd.test.ts +++ b/packages/flowsafe/src/cloudflare-fidelity.workerd.test.ts @@ -27,22 +27,22 @@ describe('FlowSafe Cloudflare fidelity', () => { expect(await readDeploymentIdentity(db())).toBeUndefined(); await Promise.all([ - seedDeploymentIdentity(db(), 'acme'), - seedDeploymentIdentity(db(), 'acme'), + seedDeploymentIdentity(db(), 'acme', 'open'), + seedDeploymentIdentity(db(), 'acme', 'open'), ]); await expect(assertDeploymentIdentity(db(), 'acme')).resolves.toBe( undefined, ); - await expect(seedDeploymentIdentity(db(), 'globex')).rejects.toThrow( - /already belongs to deployment 'acme'/, - ); + await expect( + seedDeploymentIdentity(db(), 'globex', 'open'), + ).rejects.toThrow(/already belongs to deployment 'acme'/); expect(await readDeploymentIdentity(db())).toBe('acme'); }); it('isolates D1 per test and resolves concurrent resource claims to one owner', async () => { expect(await readDeploymentIdentity(db())).toBeUndefined(); - await seedDeploymentIdentity(db(), 'acme'); + await seedDeploymentIdentity(db(), 'acme', 'open'); const store = new D1ResourceOwnershipStore(db()); const contenders: readonly ResourceOwner[] = [ diff --git a/packages/flowsafe/src/deployment-identity-protocol.ts b/packages/flowsafe/src/deployment-identity-protocol.ts index 34e9bc0..016d5cd 100644 --- a/packages/flowsafe/src/deployment-identity-protocol.ts +++ b/packages/flowsafe/src/deployment-identity-protocol.ts @@ -4,9 +4,11 @@ export type { DeploymentIdentityProtocolExecutor, DeploymentIdentityProtocolRow, DeploymentIdentityProtocolStatement, + InitialExecutionFenceState, } from '#deployment-identity-protocol'; export { assertDeploymentIdentitySecret, + assertInitialExecutionFenceState, assertValidDeploymentTag, DEPLOYMENT_ENVIRONMENT_PATTERN, DEPLOYMENT_IDENTITY_HEADER, @@ -17,6 +19,11 @@ export { DeploymentIdentityError, deploymentIdentityApplicationTables, deploymentIdentityHeaders, + EXECUTION_FENCE_DDL, + EXECUTION_FENCE_ROW_ID, + EXECUTION_FENCE_STATES, + EXECUTION_FENCE_TABLE, + INITIAL_EXECUTION_FENCE_STATES, isDeploymentEnvironment, normalizeDeploymentSentinelSql, provisionDeploymentIdentityProtocol, diff --git a/packages/flowsafe/src/do-runner/CLAUDE.md b/packages/flowsafe/src/do-runner/CLAUDE.md index 4f6ed00..3123127 100644 --- a/packages/flowsafe/src/do-runner/CLAUDE.md +++ b/packages/flowsafe/src/do-runner/CLAUDE.md @@ -2,6 +2,12 @@ - `init.ts`, `runtime.ts`, `d1-storage.ts`: import swap, execution, D1, and retention - `deployment-identity.ts`: environment binding and D1 sentinel verification +- `execution-fence.ts`: deployment-wide fence storage, CAS transitions, refusals, and admission predicates +- `start-idempotency.ts`: owner-bound start reservations, claims, replay decisions, and retention state +- `inventory.ts`: read-only drain categories, proof contract, unenumerables, and owned-table census +- `notification-predicate.ts`: import-safe due-notification predicate shared by inventory and notification storage +- `cause-chain.ts`: bounded root-cause inspection for narrowly recognized missing-table reads +- `do-status-error.ts`, `do-error-response.ts`: Durable Object refusal base class and structured HTTP rendering - `durable-object.ts`, `thread-do.ts`, `hub-do.ts`: Durable Object hosts - `path-safe-id.ts`, `memory-id.ts`, `execution-principal-header.ts`: run, memory, and execution identity - `runtime.ts`, `pubsub.ts`: resume provenance and observation state diff --git a/packages/flowsafe/src/do-runner/cause-chain.test.ts b/packages/flowsafe/src/do-runner/cause-chain.test.ts new file mode 100644 index 0000000..969d6af --- /dev/null +++ b/packages/flowsafe/src/do-runner/cause-chain.test.ts @@ -0,0 +1,146 @@ +// SPDX-License-Identifier: Apache-2.0 +// The shared cause-chain walk, tested directly rather than only through the two +// stores that consult it. +// +// It is worth its own file because both of its callers use its answer in the +// dangerous direction: "this table does not exist" is what makes a pre-0.20 +// database read as an OPEN fence, and what makes an unused key read as NO +// reservation — the answer that starts a run. The cases below are the ones a +// walker written twice gets wrong once: a cycle, a chain deeper than the bound, +// and a missing-table message that is a WRAPPER rather than the fault. + +import { describe, expect, it } from 'vitest'; + +import { findInCauseChain, missingTableReadsEmpty } from './cause-chain.js'; + +/** An error chain, innermost last. */ +function chain(...messages: string[]): Error { + let current: Error | undefined; + for (const message of [...messages].reverse()) { + current = + current === undefined + ? new Error(message) + : new Error(message, { cause: current }); + } + return current as Error; +} + +describe('findInCauseChain', () => { + it('tests only the innermost error for a rootOnly search', () => { + // #given a chain whose TOP link matches and whose root does not + const error = chain('matches', 'does not'); + + // #when / #then rootOnly answers about the fault itself, not about what + // some wrapper mentioned on the way out. + expect( + findInCauseChain(error, (link) => String(link).includes('matches'), { + rootOnly: true, + }), + ).toBe(false); + expect( + findInCauseChain(error, (link) => String(link).includes('does not'), { + rootOnly: true, + }), + ).toBe(true); + }); + + it('answers on the first matching link when rootOnly is off', () => { + // #given the same chain + const error = chain('matches', 'does not'); + + // #when / #then the any-level policy is a different claim, and the flag is + // what makes which one a caller asked for visible at its call site. + expect( + findInCauseChain(error, (link) => String(link).includes('matches'), { + rootOnly: false, + }), + ).toBe(true); + }); + + it('terminates on a cyclic chain instead of spinning', () => { + // #given two errors that cause each other — reachable from any adapter + // that re-wraps an error it is already carrying + const inner = new Error('inner'); + const outer = new Error('outer', { cause: inner }); + (inner as { cause?: unknown }).cause = outer; + + // #then it returns, and it returns the CLOSED answer: the walk reached no + // root, and an unobserved root is not evidence of anything. + expect(findInCauseChain(inner, () => true, { rootOnly: true })).toBe(false); + }); + + it('degrades closed on a chain deeper than the bound', () => { + // #given a chain longer than the depth bound, whose root WOULD match + const error = chain( + 'w1', + 'w2', + 'w3', + 'w4', + 'w5', + 'w6', + 'w7', + 'w8', + 'w9', + 'root', + ); + + // #then the root was never observed, so the answer is `false` rather than + // an unbounded walk or an optimistic guess. + expect( + findInCauseChain(error, (link) => String(link).includes('root'), { + rootOnly: true, + }), + ).toBe(false); + }); +}); + +describe('missingTableReadsEmpty', () => { + it('reads a wrapped SQLite miss as an empty table', () => { + // #given the shape an adapter produces: its own message on top, the driver + // text on `cause`. Reading only the top message here would turn a correctly + // upgraded database into a permanent 503. + const error = chain( + 'D1_ERROR: query failed', + 'no such table: flowsafe_execution_fence', + ); + + // #then + expect(missingTableReadsEmpty(error, 'flowsafe_execution_fence')).toBe( + true, + ); + }); + + it('refuses a missing-table message whose own cause is something else', () => { + // #given a fault that merely PASSED this table on its way out — a failed + // migration, an adapter reporting the last thing it saw + const error = chain( + 'no such table: flowsafe_execution_fence', + 'D1_ERROR: connection reset', + ); + + // #then not an empty table. Concluding "there is no fence" from this is the + // one answer that must never be wrong. + expect(missingTableReadsEmpty(error, 'flowsafe_execution_fence')).toBe( + false, + ); + }); + + it('refuses a miss naming a DIFFERENT table', () => { + // #given — each store asks about its own table, and a deployment can have + // one without the other + const error = new Error('no such table: flowsafe_start_idempotency'); + + // #then + expect(missingTableReadsEmpty(error, 'flowsafe_execution_fence')).toBe( + false, + ); + }); + + it('refuses an ordinary failure', () => { + // #given / #then anything that is not a missing table is a real fault, and + // both stores turn it into their own 503. + expect( + missingTableReadsEmpty(new Error('D1_ERROR: network'), 'any_table'), + ).toBe(false); + }); +}); diff --git a/packages/flowsafe/src/do-runner/cause-chain.ts b/packages/flowsafe/src/do-runner/cause-chain.ts new file mode 100644 index 0000000..e7090d1 --- /dev/null +++ b/packages/flowsafe/src/do-runner/cause-chain.ts @@ -0,0 +1,119 @@ +// SPDX-License-Identifier: Apache-2.0 +// Reading an error's `cause` chain, safely — and the one classification every +// lazily-created flowsafe table needs from it. +// +// WHY A SHARED MODULE. Two stores (the execution fence, the start reservation) +// have to answer the same question about the same kind of failure: "did this +// read fail because the table does not exist yet, or because something is +// wrong?" Both answers are load-bearing in the dangerous direction — "the table +// does not exist" is what makes a pre-0.20 database read as an OPEN fence and +// what makes an unused key read as NO reservation, and the second of those is +// the answer that STARTS A RUN. Two hand-written walkers is two chances for one +// of them to lose the depth bound, the cycle guard, or the root-only rule. +// +// This module imports NOTHING, so the modules that consult it can keep their +// own "imports only leaf modules" property. The fence and reservation store +// are both imported by surfaces that must not drag the D1 storage adapter (and +// @mastra/cloudflare-d1 with it) into their bundle. +// +// The drain inventory reads the reservation table on every sweep and needs +// the same classification: import `missingTableReadsEmpty` from HERE, not from +// start-idempotency.ts, and pass the table it is reading. + +/** + * How far down a `cause` chain any walk here looks. + * + * Bounded because a chain can be cyclic or adversarially deep, and these walks + * run on the read that fronts every gated request. A chain that does not + * terminate within the bound has NO REACHABLE ROOT, which is not the same as + * having a root that fails the predicate — see `findInCauseChain`. + */ +const MAX_CAUSE_CHAIN_DEPTH = 8; + +export interface CauseChainSearch { + /** + * `true` test ONLY the innermost error, after descending the whole chain. + * `false` test every link, and answer `true` on the first that matches. + * + * Required rather than defaulted, because the two are different CLAIMS and a + * caller that has not thought about which it wants has not thought about the + * question. Root-only says "the fault IS this thing"; any-level says "the + * fault MENTIONS this thing somewhere on its way out", which a failed + * migration or an adapter reporting the last thing it saw also satisfies. + */ + rootOnly: boolean; +} + +/** + * Walk `error.cause` and test the links this search asks for. + * + * A ROOT is a link that nothing caused. Only that link answers a `rootOnly` + * search, and the two ways a walk can end WITHOUT reaching one both answer + * `false`: + * + * A CYCLE (`a.cause = b; b.cause = a`). The walk stops on the link whose own + * cause it has already visited, and that link is NOT a root — nothing about + * it is innermost, the chain simply has no innermost. Testing it anyway + * would answer a rootOnly search on a guess, in the direction where the + * guess opens a fence or starts a second run. + * + * A CHAIN DEEPER THAN THE BOUND. No root was observed, and an unobserved + * root is not evidence of anything. + * + * Every caller here treats `false` as the closed direction, so both degrade + * the way the surrounding store degrades. + */ +export function findInCauseChain( + error: unknown, + predicate: (link: unknown) => boolean, + options: CauseChainSearch, +): boolean { + const seen = new Set(); + let current: unknown = error; + for (let depth = 0; depth < MAX_CAUSE_CHAIN_DEPTH; depth += 1) { + if (!options.rootOnly && predicate(current)) return true; + seen.add(current); + const cause = current instanceof Error ? current.cause : undefined; + if (cause === undefined || cause === null) { + // A genuine root: nothing caused this one. + return options.rootOnly ? predicate(current) : false; + } + if (seen.has(cause)) return false; + current = cause; + } + return false; +} + +/** An error's own message, however it was thrown. */ +function messageOf(error: unknown): string { + return error instanceof Error ? error.message : String(error); +} + +/** + * Does this failure mean "that table has never been created", so a read of it + * should answer EMPTY rather than fault? + * + * Matched on the MESSAGE because the structural database seams these stores use + * (`ExecutionFenceDatabase`, `StartIdempotencyDatabase`) carry no error codes — + * D1 wraps SQLite's text but preserves it. + * + * Matched at the ROOT of the cause chain and nowhere else. The chain is walked + * at all because the seam is structural: an adapter is free to wrap the + * driver's error in one of its own ("D1 query failed") and carry the SQLite + * text on `cause`, and reading only the top message there would classify a + * correctly-upgraded database as unreadable — a permanent 503 on every gated + * path. But a missing-table link whose own cause is something ELSE describes a + * fault that merely PASSED this table on its way out, and concluding "there is + * nothing here" from that is the answer that opens a fence or starts a second + * run. So the walk descends first and tests once. + */ +export function missingTableReadsEmpty(error: unknown, table: string): boolean { + return findInCauseChain( + error, + (link) => { + const message = messageOf(link); + return /no such table/i.test(message) && message.includes(table); + }, + { rootOnly: true }, + ); +} diff --git a/packages/flowsafe/src/do-runner/d1-storage.test.ts b/packages/flowsafe/src/do-runner/d1-storage.test.ts index 631d82e..7d4629a 100644 --- a/packages/flowsafe/src/do-runner/d1-storage.test.ts +++ b/packages/flowsafe/src/do-runner/d1-storage.test.ts @@ -11,6 +11,7 @@ import { sqliteUnitDatabase, } from '../../test-support/sqlite.js'; import { + createResourceOwnershipSchema, D1ResourceOwnershipStore, RESOURCE_OWNERSHIP_TABLE, type ResourceOwnershipDatabase, @@ -39,11 +40,16 @@ import { purgeExpiredThreadState, purgeExpiredThreads, purgeExpiredWorkflowRuns, + RUN_TTL_FLOWSAFE_PURGE_TABLES, type RunDeadlineCursor, type SnapshotDatabase, type SnapshotStatement, sweepExpiredRunDeadlines, } from './d1-storage.js'; +import { + START_IDEMPOTENCY_DDL, + START_IDEMPOTENCY_TABLE, +} from './start-idempotency.js'; // Domain-local result-envelope adapter for pure purge SQL units. It maps // node:sqlite's affected-row count to the structural SnapshotDatabase seam; @@ -2197,3 +2203,415 @@ describe('purgeExpiredScheduleTriggers', () => { ).toBe(0); }); }); + +// --------------------------------------------------------------------------- +// Start-reservation retention. +// +// The reservation is what makes a spent idempotency key answerable, so its +// retention has one hard rule and one soft one: +// +// HARD a reservation must NEVER be deleted while the run it names is still +// readable. Break it and the very next retry of that key mints a fresh +// run beside the live one — the exact double-execution the key was +// bought to prevent. +// SOFT a reservation must eventually be deleted, or the one table this +// deployment cannot drain grows forever. +// --------------------------------------------------------------------------- + +function createReservationTable(db: SqliteDatabase): void { + db.prepare(START_IDEMPOTENCY_DDL).run(); +} + +function seedReservation( + db: SqliteDatabase, + options: { + key: string; + runId: string; + state: 'reserved' | 'started' | 'terminal'; + updatedAt: number; + }, +): void { + db.prepare( + `INSERT INTO ${START_IDEMPOTENCY_TABLE} + (key, owner_kind, owner_id, target_kind, target_id, run_id, thread_id, + state, created_at, updated_at) + VALUES (?, 'human', 'operator-1', 'workflow', 'wf', ?, NULL, ?, ?, ?)`, + ).run( + options.key, + options.runId, + options.state, + options.updatedAt, + options.updatedAt, + ); +} + +function reservationRows( + db: SqliteDatabase, +): Array<{ key: string; run_id: string; state: string; updated_at: number }> { + return ( + db.prepare( + `SELECT key, run_id, state, updated_at FROM ${START_IDEMPOTENCY_TABLE} + ORDER BY key`, + ) as unknown as { + all(): Array<{ + key: string; + run_id: string; + state: string; + updated_at: number; + }>; + } + ).all(); +} + +describe('purgeExpiredWorkflowRuns — start reservations', () => { + it('deletes a spent reservation in the SAME batch as its run’s snapshot', async () => { + // #given a completed run past both horizons, with its key already settled + const sqlite = openSqlite(); + createSnapshotTable(sqlite); + createReservationTable(sqlite); + await createResourceOwnershipSchema(sqliteUnitDatabase(sqlite) as never); + seedRun(sqlite, { + runId: 'run-old', + status: 'success', + updatedAt: NOW - 8 * DAY_MS, + }); + seedReservation(sqlite, { + key: 'key-old', + runId: 'run-old', + state: 'terminal', + updatedAt: NOW - 8 * DAY_MS, + }); + + // #when + await purgeExpiredWorkflowRuns(sqliteUnitDatabase(sqlite) as never, { + ttlMs: 7 * DAY_MS, + resourceOwnerTable: RESOURCE_OWNERSHIP_TABLE, + startIdempotencyTable: START_IDEMPOTENCY_TABLE, + now: () => NOW, + }); + + // #then both are gone, and gone together + expect(remainingRunIds(sqlite)).toEqual([]); + expect(reservationRows(sqlite)).toEqual([]); + }); + + it('KEEPS a reservation whose horizon has not elapsed, so a late retry is told ALREADY_SETTLED', async () => { + // #given a run at the run-TTL boundary but a key-validity horizon twice as + // long — the configuration a host uses when its callers retry for longer + // than it keeps run summaries + const sqlite = openSqlite(); + createSnapshotTable(sqlite); + createReservationTable(sqlite); + await createResourceOwnershipSchema(sqliteUnitDatabase(sqlite) as never); + seedRun(sqlite, { + runId: 'run-old', + status: 'success', + updatedAt: NOW - 8 * DAY_MS, + }); + seedReservation(sqlite, { + key: 'key-old', + runId: 'run-old', + state: 'terminal', + updatedAt: NOW - 8 * DAY_MS, + }); + + // #when + await purgeExpiredWorkflowRuns(sqliteUnitDatabase(sqlite) as never, { + ttlMs: 7 * DAY_MS, + startIdempotencyTtlMs: 30 * DAY_MS, + resourceOwnerTable: RESOURCE_OWNERSHIP_TABLE, + startIdempotencyTable: START_IDEMPOTENCY_TABLE, + now: () => NOW, + }); + + // #then the snapshot is reclaimed and the reservation OUTLIVES it. That + // ordering is the whole point: a retry after this pass hits + // ALREADY_SETTLED instead of looking like a brand-new key. + expect(remainingRunIds(sqlite)).toEqual([]); + expect(reservationRows(sqlite)).toEqual([ + expect.objectContaining({ key: 'key-old', state: 'terminal' }), + ]); + }); + + it('floors the reservation horizon at the run TTL, whatever a caller asks for', async () => { + // #given a caller asking for a horizon SHORTER than run retention — a + // configuration in which a reservation would be reaped while its run is + // still readable, and the next retry of that key would start a second run + const sqlite = openSqlite(); + createSnapshotTable(sqlite); + createReservationTable(sqlite); + seedRun(sqlite, { + runId: 'run-live', + status: 'success', + updatedAt: NOW - 1 * DAY_MS, + }); + seedReservation(sqlite, { + key: 'key-live', + runId: 'run-live', + state: 'terminal', + updatedAt: NOW - 1 * DAY_MS, + }); + + // #when + await purgeExpiredWorkflowRuns(sqliteUnitDatabase(sqlite) as never, { + ttlMs: 7 * DAY_MS, + startIdempotencyTtlMs: 1, + startIdempotencyTable: START_IDEMPOTENCY_TABLE, + now: () => NOW, + }); + + // #then the run is not eligible, and neither is its reservation: the floor + // makes the dangerous configuration unreachable rather than merely unwise. + expect(remainingRunIds(sqlite)).toEqual(['run-live']); + expect(reservationRows(sqlite)).toHaveLength(1); + }); + + it('marks a reservation the terminal reconcile missed, instead of stranding it', async () => { + // #given a run that completed and was purged, but whose reservation is + // still 'started' — the shape a crash between the terminal persist and + // settleRun leaves behind + const sqlite = openSqlite(); + createSnapshotTable(sqlite); + createReservationTable(sqlite); + seedRun(sqlite, { + runId: 'run-old', + status: 'success', + updatedAt: NOW - 8 * DAY_MS, + }); + seedReservation(sqlite, { + key: 'key-stranded', + runId: 'run-old', + state: 'started', + updatedAt: NOW - 8 * DAY_MS, + }); + + // #when + await purgeExpiredWorkflowRuns(sqliteUnitDatabase(sqlite) as never, { + ttlMs: 7 * DAY_MS, + startIdempotencyTable: START_IDEMPOTENCY_TABLE, + now: () => NOW, + }); + + // #then it is terminal, its horizon re-stamped from THIS moment, and it + // survives this pass — so it is both purgeable later and out of the drain + // inventory now. + expect(reservationRows(sqlite)).toEqual([ + { + key: 'key-stranded', + run_id: 'run-old', + state: 'terminal', + updated_at: NOW, + }, + ]); + }); + + it('reaps a reservation ORPHANED by an earlier pass, once past its horizon', async () => { + // #given a reservation whose run's snapshot was purged long ago. The + // batch pairing can never see it again — its run is not in any eligible + // set — so without a sweep of its own this row would live forever. + const sqlite = openSqlite(); + createSnapshotTable(sqlite); + createReservationTable(sqlite); + seedReservation(sqlite, { + key: 'key-orphan', + runId: 'run-long-gone', + state: 'terminal', + updatedAt: NOW - 40 * DAY_MS, + }); + seedReservation(sqlite, { + key: 'key-young-orphan', + runId: 'run-also-gone', + state: 'terminal', + updatedAt: NOW - 1 * DAY_MS, + }); + + // #when + await purgeExpiredWorkflowRuns(sqliteUnitDatabase(sqlite) as never, { + ttlMs: 7 * DAY_MS, + startIdempotencyTable: START_IDEMPOTENCY_TABLE, + now: () => NOW, + }); + + // #then only the one past its horizon + expect(reservationRows(sqlite).map((row) => row.key)).toEqual([ + 'key-young-orphan', + ]); + }); + + it('never reaps an orphan candidate whose run is still readable', async () => { + // #given a reservation older than every horizon whose run STILL EXISTS — + // a live suspended run, which retention never touches + const sqlite = openSqlite(); + createSnapshotTable(sqlite); + createReservationTable(sqlite); + seedRun(sqlite, { + runId: 'run-suspended', + status: 'suspended', + updatedAt: NOW - 90 * DAY_MS, + }); + seedReservation(sqlite, { + key: 'key-suspended', + runId: 'run-suspended', + state: 'terminal', + updatedAt: NOW - 90 * DAY_MS, + }); + + // #when + await purgeExpiredWorkflowRuns(sqliteUnitDatabase(sqlite) as never, { + ttlMs: 7 * DAY_MS, + startIdempotencyTable: START_IDEMPOTENCY_TABLE, + now: () => NOW, + }); + + // #then it survives. `NOT EXISTS (snapshot)` is not an optimization — it + // is what makes the HARD rule structural rather than a consequence of + // whatever a host configured the horizon to be. + expect(reservationRows(sqlite)).toHaveLength(1); + }); + + it('sweeps orphans on the strict side of the horizon, and never one whose snapshot survives', async () => { + // #given the three rows the sweep's predicate has to separate, in ONE pass + // so they are judged by the same cutoff. With no `startIdempotencyTtlMs` + // the horizon is the run TTL, so the boundary is exactly NOW - 7 days and + // the comparison is `updated_at < cutoff` — strict, because a row AT the + // cutoff has not yet outlived it. + const sqlite = openSqlite(); + createSnapshotTable(sqlite); + createReservationTable(sqlite); + const cutoff = NOW - 7 * DAY_MS; + seedReservation(sqlite, { + key: 'key-at-cutoff', + runId: 'run-gone-a', + state: 'terminal', + updatedAt: cutoff, + }); + seedReservation(sqlite, { + key: 'key-past-cutoff', + runId: 'run-gone-b', + state: 'terminal', + updatedAt: cutoff - 1, + }); + // Old enough to sweep on age alone, but its run is still readable — a + // suspended run, which run retention never reclaims. + seedRun(sqlite, { + runId: 'run-still-here', + status: 'suspended', + updatedAt: NOW - 90 * DAY_MS, + }); + seedReservation(sqlite, { + key: 'key-with-snapshot', + runId: 'run-still-here', + state: 'terminal', + updatedAt: NOW - 90 * DAY_MS, + }); + + // #when + await purgeExpiredWorkflowRuns(sqliteUnitDatabase(sqlite) as never, { + ttlMs: 7 * DAY_MS, + startIdempotencyTable: START_IDEMPOTENCY_TABLE, + now: () => NOW, + }); + + // #then exactly the row PAST the horizon is gone. The at-cutoff row is the + // boundary this test exists for: a `<=` here would reap a key on the last + // instant it is still meant to answer ALREADY_SETTLED, and the retry that + // arrives in that instant would start a second run. The snapshot-backed row + // survives on `NOT EXISTS`, whatever its age, which is the HARD rule. + expect(reservationRows(sqlite).map((row) => row.key)).toEqual([ + 'key-at-cutoff', + 'key-with-snapshot', + ]); + }); + + it('still purges runs on a deployment where no key has ever been used', async () => { + // #given the reservation table wired but never created — its DDL is lazy, + // so a deployment on which nobody used a key has none. A batch naming a + // missing table fails as ONE TRANSACTION, which would take run retention + // down with it. + const sqlite = openSqlite(); + createSnapshotTable(sqlite); + await createResourceOwnershipSchema(sqliteUnitDatabase(sqlite) as never); + seedRun(sqlite, { + runId: 'run-old', + status: 'success', + updatedAt: NOW - 8 * DAY_MS, + }); + + // #when + const deleted = await purgeExpiredWorkflowRuns( + sqliteUnitDatabase(sqlite) as never, + { + ttlMs: 7 * DAY_MS, + resourceOwnerTable: RESOURCE_OWNERSHIP_TABLE, + startIdempotencyTable: START_IDEMPOTENCY_TABLE, + now: () => NOW, + }, + ); + + // #then retention is enforced anyway: an absent table holds no reservation + // to reap, which is not a reason to stop reclaiming runs. + expect(deleted).toBe(1); + expect(remainingRunIds(sqlite)).toEqual([]); + }); + + it('pairs reservations on the artifact path too', async () => { + // #given the per-run path a host with R2 artifacts takes — a different + // batch, and therefore a second place the pairing could have been missed + const sqlite = openSqlite(); + createSnapshotTable(sqlite); + createReservationTable(sqlite); + seedRun(sqlite, { + runId: 'run-old', + status: 'success', + updatedAt: NOW - 8 * DAY_MS, + }); + seedReservation(sqlite, { + key: 'key-old', + runId: 'run-old', + state: 'terminal', + updatedAt: NOW - 8 * DAY_MS, + }); + + // #when + await purgeExpiredWorkflowRuns(sqliteUnitDatabase(sqlite) as never, { + ttlMs: 7 * DAY_MS, + artifactStore: { deleteRun: async () => 0 }, + startIdempotencyTable: START_IDEMPOTENCY_TABLE, + now: () => NOW, + }); + + // #then + expect(remainingRunIds(sqlite)).toEqual([]); + expect(reservationRows(sqlite)).toEqual([]); + }); + + it('refuses a reservation table name that is not a safe SQL identifier', async () => { + // #given — the name is interpolated into every statement above + const sqlite = openSqlite(); + createSnapshotTable(sqlite); + + // #when / #then + await expect( + purgeExpiredWorkflowRuns(sqliteUnitDatabase(sqlite) as never, { + ttlMs: DAY_MS, + startIdempotencyTable: 'reservations; DROP TABLE x', + }), + ).rejects.toThrow(/safe SQL identifier/); + }); +}); + +describe('RUN_TTL_FLOWSAFE_PURGE_TABLES', () => { + it('names the production constants, not literals, so a rename fails here', () => { + // #given — the flowsafe-owned half of what run retention deletes from. + // It is separate from RUN_TTL_PURGE_TABLES because the schema guard's + // biconditional is over the `mastra_%` inventory: folding ours in would + // make that guard assert an equality it cannot mean. + // + // #then each entry is the EXPORTED name its purge statement interpolates. + // A rename of either table changes both sides at once, so this cannot drift + // the way a copied literal would. + expect([...RUN_TTL_FLOWSAFE_PURGE_TABLES].sort()).toEqual( + [RESOURCE_OWNERSHIP_TABLE, START_IDEMPOTENCY_TABLE].sort(), + ); + }); +}); diff --git a/packages/flowsafe/src/do-runner/d1-storage.ts b/packages/flowsafe/src/do-runner/d1-storage.ts index ae0e692..d6d1703 100644 --- a/packages/flowsafe/src/do-runner/d1-storage.ts +++ b/packages/flowsafe/src/do-runner/d1-storage.ts @@ -14,6 +14,7 @@ import { import type { D1DatabaseBinding } from './cf-types.js'; import { isPathSafeId } from './path-safe-id.js'; +import { START_IDEMPOTENCY_TABLE } from './start-idempotency.js'; import { validateTablePrefix } from './table-prefix.js'; export interface D1StorageOptions { @@ -61,10 +62,19 @@ export function createD1Storage( }); } -// Mastra workflow terminals plus FlowSafe's lifecycle-owned terminals. -// Deleting a live run (running/suspended/waiting/pending/paused) would kill a -// pending approval, so only these are ever purged. -const TERMINAL_STATUSES = [ +/** + * Mastra workflow terminals plus FlowSafe's lifecycle-owned terminals. + * Deleting a live run (running/suspended/waiting/pending/paused) would kill a + * pending approval, so only these are ever purged. + * + * Exported because retention and the drain inventory must agree on the word + * "terminal" to the letter. The purge deletes what this set matches; the + * inventory counts what it does NOT, and a run that is terminal to one and live + * to the other is either a row the purge reaps while the inventory still calls + * it work, or — worse — a run the inventory declares finished while it is still + * executing, which is exactly the reading a migration would act on. + */ +export const RUN_TERMINAL_STATUSES = [ 'success', 'failed', 'tripwire', @@ -75,6 +85,31 @@ const TERMINAL_STATUSES = [ 'timed_out', ] as const; +/** + * The SQL that decides one snapshot row is TERMINAL, with `?` for each entry of + * RUN_TERMINAL_STATUSES in order. + * + * A shared FRAGMENT rather than a shared list because the rule is not "the + * status is in the set": a run that reached 'cancelled' or 'timed_out' is + * terminal only once its lifecycle cleanup stamped `cleanupCompletedAt`, and + * before that it is still executing compensation. Two hand-written copies of + * that carve-out are two chances for one of them to answer "finished" for a run + * that is mid-cleanup — the retention purge would delete a live run's snapshot, + * and the drain inventory would report a deployment empty while it still runs + * work. The caller supplies the `json_valid` guard, because it decides which + * way an unclassifiable row should fail. + */ +export const RUN_TERMINAL_SNAPSHOT_SQL = `json_extract(snapshot, '$.status') IN (${RUN_TERMINAL_STATUSES.map( + () => '?', +).join(', ')}) + AND ( + json_extract(snapshot, '$.status') NOT IN ('cancelled', 'timed_out') + OR json_extract( + snapshot, + '$.requestContext."flowsafe.runLifecycle".terminal.cleanupCompletedAt' + ) IS NOT NULL + )`; + const DEADLINE_LIVE_STATUSES = [ 'running', 'waiting', @@ -310,6 +345,37 @@ export interface PurgeExpiredRunsOptions { * resource registry omit it. */ resourceOwnerTable?: string; + /** + * The start-reservation table (`flowsafe_start_idempotency`). When supplied, + * this purge is also what keeps idempotency keys finite. + * + * Wired the same way `resourceOwnerTable` is — by name, from the composed + * Flowsafe Worker — for the same reason: this module owns the SQL of run + * retention, and a reservation must be reaped in the same transaction that + * removes the run it points at, never by a second sweep that could interleave + * with it. + */ + startIdempotencyTable?: string; + /** + * How long a spent idempotency key stays answerable after its run settled — + * the KEY-VALIDITY HORIZON, and the only tuning decision this feature has. + * + * Until it elapses, a retry of a completed run is told ALREADY_SETTLED. After + * it, the reservation is gone and the same key reads as brand new, so a retry + * would START A SECOND RUN. That is the whole reason this exists as its own + * knob rather than riding `ttlMs`: a host whose callers retry for longer than + * its run retention (an overnight batch re-run, a queue with a multi-day + * redrive) needs keys to outlive summaries, and a host whose keys are minted + * per HTTP request does not. + * + * DEFAULTS TO `ttlMs`, and is floored at it: a reservation shorter-lived than + * the snapshot it guards would be deleted while its run is still readable, + * and the very next retry would mint a fresh run alongside the live one — the + * exact double-execution this feature exists to prevent. A caller asking for + * less gets `ttlMs`, silently, because there is no configuration in which the + * smaller number is what anybody meant. + */ + startIdempotencyTtlMs?: number; /** * Runs processed per call. Artifact-paired path: default 100 — the purge duty's * subrequest-budget guard, same batching as any batched reaper. Each @@ -327,22 +393,61 @@ export interface PurgeExpiredRunsOptions { } /** - * The tables `purgeExpiredWorkflowRuns` deletes from under the run TTL — the - * production anchor the schema guard cross-checks every `run-ttl` retention - * declaration against. The guard reads THIS, not a literal copied into the - * test, so a purge that changes what it targets and a guard that still + * The MASTRA-OWNED tables `purgeExpiredWorkflowRuns` deletes from under the run + * TTL — the production anchor the schema guard cross-checks every `run-ttl` + * retention declaration against. The guard reads THIS, not a literal copied + * into the test, so a purge that changes what it targets and a guard that still * blesses the old set cannot drift apart silently. + * + * Mastra-owned specifically: the purge ALSO deletes from flowsafe's own + * registries when a caller wires them, and those live in + * RUN_TTL_FLOWSAFE_PURGE_TABLES below rather than here — see its note for why + * the two sets are not one. */ export const RUN_TTL_PURGE_TABLES: readonly string[] = [ 'mastra_workflow_snapshot', ]; +/** + * The resource-ownership registry's table, named here rather than imported from + * the store that creates it (approval-api/resource-ownership.ts). + * + * The layering forbids the import: do-runner may reach approval-api only + * through its declared leaves, and the ownership store is not one — it is built + * ON do-runner. So this file has always carried the name as a literal inside + * RUN_TTL_FLOWSAFE_PURGE_TABLES; giving it a name adds no second home, it names + * the one that was already here, and lets the drain inventory read the registry + * without a third copy. The census test crosses it against + * RESOURCE_OWNERSHIP_TABLE, which is the only place the two can be compared. + */ +export const RESOURCE_OWNER_TABLE = 'flowsafe_resource_owners'; + +/** + * The FLOWSAFE-owned tables this purge also deletes from when the caller wires + * them, and the reason they are not in the list above. + * + * `RUN_TTL_PURGE_TABLES` is cross-checked against the `mastra_%` inventory in + * mastra-schema-guard.test.ts — its job is to catch a @mastra/core bump that + * changes what run retention targets. These two are ours, they are optional + * (a lower-level caller without the registries omits both), and they are + * deleted on a DIFFERENT predicate: `flowsafe_resource_owners` when its run's + * last snapshot is gone, `flowsafe_start_idempotency` when its reservation is + * settled AND past the key-validity horizon. Folding them into the Mastra + * anchor would make that guard assert an equality it cannot mean. + */ +export const RUN_TTL_FLOWSAFE_PURGE_TABLES: readonly string[] = [ + RESOURCE_OWNER_TABLE, + START_IDEMPOTENCY_TABLE, +]; + /** * Data-retention purge: deletes TERMINAL runs (success/failed/tripwire/ * canceled/bailed/skipped and cleanup-complete cancelled/timed_out) whose * updatedAt is older than the TTL from * mastra_workflow_snapshot — and, when `artifactStore` is wired, each purged - * run's R2 artifacts with its row. Live runs (running/suspended/waiting/ + * run's R2 artifacts with its row, plus (when their tables are wired) the run's + * ownership row and its spent start reservation, each in the SAME transaction + * as the snapshot delete. Live runs (running/suspended/waiting/ * pending/paused) are never touched — expiring a suspended run would kill a * pending approval. A missing snapshot table reads as zero purgeable runs * (Mastra creates it lazily with the first persisted run). TTL enforcement @@ -359,7 +464,6 @@ export async function purgeExpiredWorkflowRuns( // (persistWorkflowSnapshot serializes via toISOString), so lexicographic // < against an ISO cutoff is a correct timestamp comparison. const cutoff = new Date(now() - options.ttlMs).toISOString(); - const placeholders = TERMINAL_STATUSES.map(() => '?').join(', '); // json_extract throws on malformed JSON and would abort the WHOLE delete — // one corrupt row must not stop every valid terminal row from being // reclaimed. The CASE guard (not `AND json_valid(...)`) is load-bearing: @@ -370,14 +474,7 @@ export async function purgeExpiredWorkflowRuns( const eligible = `updatedAt < ? AND CASE WHEN json_valid(snapshot) THEN ( - json_extract(snapshot, '$.status') IN (${placeholders}) - AND ( - json_extract(snapshot, '$.status') NOT IN ('cancelled', 'timed_out') - OR json_extract( - snapshot, - '$.requestContext."flowsafe.runLifecycle".terminal.cleanupCompletedAt' - ) IS NOT NULL - ) + ${RUN_TERMINAL_SNAPSHOT_SQL} ) ELSE 0 END`; const resourceOwnerTable = options.resourceOwnerTable; @@ -387,12 +484,173 @@ export async function purgeExpiredWorkflowRuns( ) { throw new Error('resourceOwnerTable must be a safe SQL identifier'); } - const batch = resourceOwnerTable ? db.batch?.bind(db) : undefined; - if (resourceOwnerTable && !batch) { + const startIdempotencyTable = options.startIdempotencyTable; + if ( + startIdempotencyTable !== undefined && + !/^[A-Za-z_][A-Za-z0-9_]*$/.test(startIdempotencyTable) + ) { + throw new Error('startIdempotencyTable must be a safe SQL identifier'); + } + // Floored at the run TTL, never below it — see startIdempotencyTtlMs. A + // reservation deleted while its run is still readable would let the next + // retry of that key start a SECOND run beside the live one. + const reservationCutoff = + now() - + Math.max(options.startIdempotencyTtlMs ?? options.ttlMs, options.ttlMs); + const batch = + resourceOwnerTable || startIdempotencyTable + ? db.batch?.bind(db) + : undefined; + if ((resourceOwnerTable || startIdempotencyTable) && !batch) { throw new Error( 'purgeExpiredWorkflowRuns requires database.batch() for atomic owner cleanup', ); } + /** + * The two reservation statements that ride a snapshot delete, in order. + * + * They run INSIDE the same `batch()` as the snapshot's own DELETE and AFTER + * it, which is what makes the pairing atomic: by the time these execute, the + * runs named here have no snapshot in this transaction, so neither statement + * can act on a reservation whose run is still readable. + * + * 1. DELETE the reservations already past the horizon. This is the pairing + * the design asks for: a spent key and the run it named leave together. + * 2. MARK the rest terminal. A reservation still inside its horizon must + * survive — that is what makes a late retry ALREADY_SETTLED rather than a + * fresh start — but its run is gone, so it is settled by definition. This + * also HEALS the reconcile a crash between a run's terminal persist and + * `settleRun` would have lost, and re-stamps `updated_at` so the horizon + * is measured from a point at which the reservation is definitely spent. + * + * `state <> 'terminal'` also settles `reserved` rows, not just `started` + * ones. The ordinary lifecycle should not produce one here — a run only + * persists a snapshot after its claim, so a row this statement can see is + * normally `started` — and the point is that this statement does not + * depend on that. It is selected by RUN, and every run it names has just + * lost its snapshot in this same transaction, so whatever left the row + * un-claimed (a released claim, a hand-edited row, a caller yet to be + * written), the run it names is gone and the key cannot be worth starting + * again. Settling too eagerly costs a retry a refusal it can resolve with + * a fresh key; leaving a row readable as `reserved` after its run is + * unreadable costs a second run of work that already completed. + */ + /** + * Whether the reservation table has been seen to exist this pass. + * + * It is created lazily by the first `reserve()`, so a deployment on which no + * idempotency key has ever been used has none — and a batch naming a missing + * table fails as ONE TRANSACTION, taking the snapshot delete down with it. + * That would turn "this host wired reservations and nobody has used one yet" + * into "run retention is silently unenforced", so the first such failure + * retries the batch WITHOUT the reservation statements and the pass carries + * on with the pairing disabled. Nothing is lost: a table that does not exist + * holds no reservation to reap. + */ + let reservationsUnavailable = false; + const reservationStatements = ( + runIds: readonly string[], + ): SnapshotStatement[] => { + if ( + !startIdempotencyTable || + reservationsUnavailable || + runIds.length === 0 + ) { + return []; + } + const placeholders = runIds.map(() => '?').join(', '); + return [ + db + .prepare( + `DELETE FROM ${startIdempotencyTable} + WHERE run_id IN (${placeholders}) + AND state = 'terminal' AND updated_at < ?`, + ) + .bind(...runIds, reservationCutoff), + db + .prepare( + `UPDATE ${startIdempotencyTable} + SET state = 'terminal', updated_at = ? + WHERE run_id IN (${placeholders}) AND state <> 'terminal'`, + ) + .bind(now(), ...runIds), + ]; + }; + /** + * Reap the reservations that OUTLIVED their snapshot. + * + * The paired statements above only ever see runs whose snapshot is expiring + * in THIS pass, and a reservation is meant to survive that moment — the whole + * point of a horizon longer than run retention is that a late retry still + * finds ALREADY_SETTLED after the summary is gone. Which means the pairing + * alone can never delete those rows: by the time they are old enough, their + * snapshot has been gone for passes and nothing re-visits them. This sweep is + * what keeps the table finite, and without it the reservation table would be + * the one piece of this deployment's state that only ever grows. + * + * `NOT EXISTS (snapshot)` is not an optimization — it is the safety predicate + * that makes this sweep structurally unable to delete a reservation whose run + * is still readable, whatever a caller configured the horizon to be. The + * LIMIT rides a rowid subselect for the same reason every other purge here + * does: plain `DELETE ... LIMIT` needs a SQLite compile-time option D1 does + * not guarantee. + */ + /** + * Run a snapshot-delete batch, retrying once without the reservation + * statements if the reservation table turns out not to exist yet. + * + * `build(withReservations)` rather than a prepared array, because D1 + * statements are single-use once run: a retry has to re-prepare. + */ + const runPurgeBatch = async ( + build: (withReservations: boolean) => SnapshotStatement[], + ): Promise => { + if (!batch) throw new Error('purgeExpiredWorkflowRuns: batch unavailable'); + try { + return await batch(build(true)); + } catch (error) { + if ( + startIdempotencyTable === undefined || + reservationsUnavailable || + !isMissingTable(error, startIdempotencyTable) + ) { + throw error; + } + reservationsUnavailable = true; + return batch(build(false)); + } + }; + const sweepOrphanedStartReservations = async (): Promise => { + if (!startIdempotencyTable || reservationsUnavailable) return; + try { + await db + .prepare( + `DELETE FROM ${startIdempotencyTable} + WHERE rowid IN ( + SELECT r.rowid FROM ${startIdempotencyTable} AS r + WHERE r.state = 'terminal' AND r.updated_at < ? + AND NOT EXISTS ( + SELECT 1 FROM ${prefix}mastra_workflow_snapshot AS s + WHERE s.run_id = r.run_id + ) + LIMIT ? + )`, + ) + .bind(reservationCutoff, options.limit ?? 1000) + .run(); + } catch (error) { + // Either table may legitimately not exist yet: the reservation table is + // created by the first reserve() and the snapshot table by the first run. + // Neither absence is a fault, and neither leaves anything to reap. + if ( + isMissingTable(error, startIdempotencyTable) || + isMissingTable(error, `${prefix}mastra_workflow_snapshot`) + ) { + return; + } + throw error; + } + }; if (options.artifactStore) { // Artifact-paired path, run by run: artifacts BEFORE the row, because @@ -411,7 +669,7 @@ export async function purgeExpiredWorkflowRuns( WHERE ${eligible} LIMIT ?`, ) - .bind(cutoff, ...TERMINAL_STATUSES, options.limit ?? 100) + .bind(cutoff, ...RUN_TERMINAL_STATUSES, options.limit ?? 100) .all<{ workflow_name: string; run_id: string }>()); } catch (error) { if (!isMissingTable(error, `${prefix}mastra_workflow_snapshot`)) @@ -439,31 +697,48 @@ export async function purgeExpiredWorkflowRuns( } // Re-checking eligibility keys the delete to the row the SELECT saw; // terminal is absorbing, so this is belt-and-braces, not a race fix. - const deleteSnapshot = db - .prepare( - `DELETE FROM ${prefix}mastra_workflow_snapshot - WHERE workflow_name = ? AND run_id = ? AND ${eligible}`, - ) - .bind(row.workflow_name, row.run_id, cutoff, ...TERMINAL_STATUSES); - if (resourceOwnerTable && batch) { - const [result] = await batch([ - deleteSnapshot, - db - .prepare( - `DELETE FROM ${resourceOwnerTable} + // + // A FACTORY, not one prepared statement: D1 statements are single-use + // once run, and `runPurgeBatch` re-prepares its whole batch when the + // reservation table turns out not to exist. Written once so the batch and + // non-batch paths cannot drift onto different delete predicates. + const deleteSnapshot = (): SnapshotStatement => + db + .prepare( + `DELETE FROM ${prefix}mastra_workflow_snapshot + WHERE workflow_name = ? AND run_id = ? AND ${eligible}`, + ) + .bind( + row.workflow_name, + row.run_id, + cutoff, + ...RUN_TERMINAL_STATUSES, + ); + if (batch) { + const [result] = await runPurgeBatch((withReservations) => [ + deleteSnapshot(), + ...(resourceOwnerTable + ? [ + db + .prepare( + `DELETE FROM ${resourceOwnerTable} WHERE resource_kind = 'run' AND resource_id = ? AND NOT EXISTS ( SELECT 1 FROM ${prefix}mastra_workflow_snapshot WHERE run_id = ? )`, - ) - .bind(row.run_id, row.run_id), + ) + .bind(row.run_id, row.run_id), + ] + : []), + ...(withReservations ? reservationStatements([row.run_id]) : []), ]); deleted += d1Changes(result); } else { - deleted += d1Changes(await deleteSnapshot.run()); + deleted += d1Changes(await deleteSnapshot().run()); } } + await sweepOrphanedStartReservations(); if (failures.length > 0) { throw new Error( `purgeExpiredWorkflowRuns: artifact deletion failed for ${failures.length} of ${rows.length} eligible run(s), the rest were purged (${failures @@ -481,8 +756,9 @@ export async function purgeExpiredWorkflowRuns( // limits and then fail the same way on EVERY firing — retention silently // unenforced, the exact wedge the one-duty-per-alarm split exists to avoid. The // shrinking eligible set is the cursor: a backlog drains across firings. + let deleted: number; try { - if (resourceOwnerTable && batch) { + if (batch) { // D1 accepts at most 100 bound parameters per statement. Keep the exact // rowid + owner-id transaction below under that limit even when a caller // requests a larger generic purge batch. @@ -494,51 +770,64 @@ export async function purgeExpiredWorkflowRuns( WHERE ${eligible} LIMIT ?`, ) - .bind(cutoff, ...TERMINAL_STATUSES, limit) + .bind(cutoff, ...RUN_TERMINAL_STATUSES, limit) .all<{ rowid: number; run_id: string }>(); - if (selected.results.length === 0) return 0; + if (selected.results.length === 0) { + // No eligible run this pass, but reservations left behind by EARLIER + // passes still age out — so the orphan sweep runs before the return. + await sweepOrphanedStartReservations(); + return 0; + } const rowIds = selected.results.map((row) => row.rowid); const runIds = [...new Set(selected.results.map((row) => row.run_id))]; const rowPlaceholders = rowIds.map(() => '?').join(', '); const runPlaceholders = runIds.map(() => '?').join(', '); - const [result] = await batch([ + const [result] = await runPurgeBatch((withReservations) => [ db .prepare( `DELETE FROM ${prefix}mastra_workflow_snapshot WHERE rowid IN (${rowPlaceholders}) AND ${eligible}`, ) - .bind(...rowIds, cutoff, ...TERMINAL_STATUSES), - db - .prepare( - `DELETE FROM ${resourceOwnerTable} + .bind(...rowIds, cutoff, ...RUN_TERMINAL_STATUSES), + ...(resourceOwnerTable + ? [ + db + .prepare( + `DELETE FROM ${resourceOwnerTable} WHERE resource_kind = 'run' AND resource_id IN (${runPlaceholders}) AND resource_id NOT IN ( SELECT run_id FROM ${prefix}mastra_workflow_snapshot )`, - ) - .bind(...runIds), + ) + .bind(...runIds), + ] + : []), + ...(withReservations ? reservationStatements(runIds) : []), ]); - return d1Changes(result); - } - return d1Changes( - await db - .prepare( - `DELETE FROM ${prefix}mastra_workflow_snapshot + deleted = d1Changes(result); + } else { + deleted = d1Changes( + await db + .prepare( + `DELETE FROM ${prefix}mastra_workflow_snapshot WHERE rowid IN ( SELECT rowid FROM ${prefix}mastra_workflow_snapshot WHERE ${eligible} LIMIT ? )`, - ) - .bind(cutoff, ...TERMINAL_STATUSES, options.limit ?? 1000) - .run(), - ); + ) + .bind(cutoff, ...RUN_TERMINAL_STATUSES, options.limit ?? 1000) + .run(), + ); + } } catch (error) { if (!isMissingTable(error, `${prefix}mastra_workflow_snapshot`)) throw error; return 0; } + await sweepOrphanedStartReservations(); + return deleted; } export interface PurgeExpiredThreadsOptions { @@ -777,6 +1066,21 @@ const BACKGROUND_TASK_FAILED_STATUSES = [ 'timed_out', ] as const; +/** + * Every background-task status this purge is willing to reap — the two TTL + * windows' sets together, and therefore the package's definition of a SETTLED + * task. + * + * The drain inventory reads it as the complement it needs: a task whose status + * is not here is still work, whether it is queued, running, or suspended + * awaiting a webhook. Deriving both from one list is what stops "terminal" + * meaning one thing to retention and another to a migration's emptiness proof. + */ +export const BACKGROUND_TASK_TERMINAL_STATUSES: readonly string[] = [ + 'completed', + ...BACKGROUND_TASK_FAILED_STATUSES, +]; + /** * Background-task TTL cleanup: deletes terminal task * rows from `mastra_background_tasks` once their `completedAt` is older than the diff --git a/packages/flowsafe/src/do-runner/deployment-identity.test.ts b/packages/flowsafe/src/do-runner/deployment-identity.test.ts index b007120..9e9800f 100644 --- a/packages/flowsafe/src/do-runner/deployment-identity.test.ts +++ b/packages/flowsafe/src/do-runner/deployment-identity.test.ts @@ -1,6 +1,11 @@ // SPDX-License-Identifier: Apache-2.0 import { describe, expect, it } from 'vitest'; +import { + DEPLOYMENT_SENTINEL_DDL, + EXECUTION_FENCE_ROW_ID, + EXECUTION_FENCE_TABLE, +} from '#deployment-identity-protocol'; import { openSqlite, sqliteUnitDatabase } from '../../test-support/sqlite.js'; import { assertDeploymentIdentity, @@ -13,6 +18,7 @@ import { deploymentIdentityHeaders, ensureDeploymentIdentity, ensureDeploymentIdentityBindings, + type InitialExecutionFenceState, readDeploymentIdentity, seedDeploymentIdentity, verifyDurableObjectDeploymentIdentity, @@ -125,11 +131,11 @@ describe('deployment identity provisioning', () => { expect(await readDeploymentIdentity(db)).toBeUndefined(); await Promise.all([ - seedDeploymentIdentity(db, 'acme'), - seedDeploymentIdentity(db, 'acme'), + seedDeploymentIdentity(db, 'acme', 'open'), + seedDeploymentIdentity(db, 'acme', 'open'), ]); expect(await readDeploymentIdentity(db)).toBe('acme'); - await expect(seedDeploymentIdentity(db, 'globex')).rejects.toThrow( + await expect(seedDeploymentIdentity(db, 'globex', 'open')).rejects.toThrow( /already belongs to deployment 'acme'/, ); expect(await readDeploymentIdentity(db)).toBe('acme'); @@ -146,7 +152,7 @@ describe('deployment identity provisioning', () => { .run(); expect(await readDeploymentIdentity(db)).toBeUndefined(); - await seedDeploymentIdentity(db, 'acme'); + await seedDeploymentIdentity(db, 'acme', 'open'); expect(await readDeploymentIdentity(db)).toBe('acme'); }); @@ -154,6 +160,7 @@ describe('deployment identity provisioning', () => { const base = sqliteDatabase(); const preparedQueries: string[] = []; const insertBindings: unknown[][] = []; + const fenceBindings: unknown[][] = []; const db: DeploymentIdentityDatabase = { prepare(query) { preparedQueries.push(query); @@ -161,7 +168,16 @@ describe('deployment identity provisioning', () => { let bound = prepared; const statement: DeploymentIdentityStatement = { bind(...values) { - if (query.startsWith('INSERT OR IGNORE')) { + // Dispatch on the INSERT TARGET: the ownership insert names the + // fence table inside its exclusion list, so a substring test would + // count it twice. + if ( + query.startsWith(`INSERT OR IGNORE INTO ${EXECUTION_FENCE_TABLE}`) + ) { + fenceBindings.push(values); + } else if ( + query.startsWith('INSERT OR IGNORE INTO flowsafe_deployment') + ) { insertBindings.push(values); } bound = prepared.bind(...values); @@ -174,10 +190,10 @@ describe('deployment identity provisioning', () => { }, }; - await seedDeploymentIdentity(db, 'acme'); + await seedDeploymentIdentity(db, 'acme', 'migration-locked'); const insert = preparedQueries.find((query) => - query.startsWith('INSERT OR IGNORE'), + query.startsWith('INSERT OR IGNORE INTO flowsafe_deployment'), ); expect(insert).toContain('SELECT 1, ?, ?'); expect(insert).not.toContain("'acme'"); @@ -186,14 +202,90 @@ describe('deployment identity provisioning', () => { expect(insertBindings[0]?.[1]).toMatch( /^\d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2}\.\d{3}Z$/, ); + + // The fence row rides the same bound-parameter boundary — the caller's + // chosen state reaches D1 as a binding, never interpolated into SQL. + const fenceInsert = preparedQueries.find((query) => + query.startsWith(`INSERT OR IGNORE INTO ${EXECUTION_FENCE_TABLE}`), + ); + expect(fenceInsert).toContain('VALUES (?, ?, NULL, NULL, ?)'); + expect(fenceInsert).not.toContain("'migration-locked'"); + expect(fenceBindings).toHaveLength(1); + expect(fenceBindings[0]?.[0]).toBe(EXECUTION_FENCE_ROW_ID); + expect(fenceBindings[0]?.[1]).toBe('migration-locked'); + expect(fenceBindings[0]?.[2]).toMatch(/^\d+$/); + + // ... and lands as the state that was asked for, on the single fixed row. + const fenceRows = await base + .prepare( + `SELECT id, state, proof_key, proof_run_id FROM ${EXECUTION_FENCE_TABLE}`, + ) + .all(); + expect(fenceRows.results).toEqual([ + { + id: EXECUTION_FENCE_ROW_ID, + state: 'migration-locked', + proof_key: null, + proof_run_id: null, + }, + ]); + }); + + it('never reopens a fence an operator closed when provisioning re-runs', async () => { + const db = sqliteDatabase(); + await seedDeploymentIdentity(db, 'acme', 'migration-locked'); + + // A re-provision asking for 'open' must leave the closed fence alone: the + // row is INSERT-if-absent, not an upsert. + await seedDeploymentIdentity(db, 'acme', 'open'); + + const rows = await db + .prepare(`SELECT state FROM ${EXECUTION_FENCE_TABLE}`) + .all<{ state: string }>(); + expect(rows.results).toEqual([{ state: 'migration-locked' }]); + }); + + it('heals a fence row an interrupted provisioning pass never wrote', async () => { + const db = sqliteDatabase(); + // Ownership stamped, fence absent: the residue of a crash between the two + // writes, which the already-owned early return has to repair. + await db.prepare(DEPLOYMENT_SENTINEL_DDL).run(); + await db + .prepare( + 'INSERT INTO flowsafe_deployment (id, tenant_tag, provisioned_at) VALUES (1, ?, ?)', + ) + .bind('acme', new Date(0).toISOString()) + .run(); + + await seedDeploymentIdentity(db, 'acme', 'migration-locked'); + + const rows = await db + .prepare(`SELECT state FROM ${EXECUTION_FENCE_TABLE}`) + .all<{ state: string }>(); + expect(rows.results).toEqual([{ state: 'migration-locked' }]); + }); + + it('refuses a fence state that is not a legal birth state', async () => { + const db = sqliteDatabase(); + for (const state of ['draining', 'proof-only', 'open ', '']) { + await expect( + seedDeploymentIdentity( + db, + 'acme', + state as unknown as InitialExecutionFenceState, + ), + ).rejects.toThrow(/must be one of open, migration-locked/); + } + // Nothing was stamped: the state is validated before the first statement. + expect(await readDeploymentIdentity(db)).toBeUndefined(); }); it('refuses malformed provisioning input and malformed sentinel content', async () => { const db = sqliteDatabase(); - await expect(seedDeploymentIdentity(db, 'ACME')).rejects.toBeInstanceOf( - DeploymentIdentityError, - ); - await seedDeploymentIdentity(db, 'acme'); + await expect( + seedDeploymentIdentity(db, 'ACME', 'open'), + ).rejects.toBeInstanceOf(DeploymentIdentityError); + await seedDeploymentIdentity(db, 'acme', 'open'); await db .prepare('UPDATE flowsafe_deployment SET tenant_tag = ? WHERE id = 1') .bind('ACME') @@ -204,7 +296,7 @@ describe('deployment identity provisioning', () => { await expect(assertDeploymentIdentity(db, 'acme')).rejects.toThrow( /tenant_tag is malformed/, ); - await expect(seedDeploymentIdentity(db, 'acme')).rejects.toThrow( + await expect(seedDeploymentIdentity(db, 'acme', 'open')).rejects.toThrow( /tenant_tag is malformed/, ); }); @@ -273,7 +365,7 @@ describe('deployment identity provisioning', () => { ])('refuses to adopt an unowned database containing %s', async (table) => { const db = sqliteDatabase(); await db.prepare(`CREATE TABLE ${table} (id TEXT PRIMARY KEY)`).run(); - await expect(seedDeploymentIdentity(db, 'acme')).rejects.toThrow( + await expect(seedDeploymentIdentity(db, 'acme', 'open')).rejects.toThrow( /unowned database already contains application tables/, ); }); @@ -286,7 +378,7 @@ describe('deployment identity provisioning', () => { ])('does not treat near-system table %s as D1-owned', async (table) => { const db = sqliteDatabase(); await db.prepare(`CREATE TABLE ${table} (id TEXT PRIMARY KEY)`).run(); - await expect(seedDeploymentIdentity(db, 'acme')).rejects.toThrow( + await expect(seedDeploymentIdentity(db, 'acme', 'open')).rejects.toThrow( /unowned database already contains application tables/, ); }); @@ -297,7 +389,7 @@ describe('deployment identity provisioning', () => { ])('allows the exact D1-owned %s table', async (table) => { const db = sqliteDatabase(); await db.prepare(`CREATE TABLE ${table} (key TEXT PRIMARY KEY)`).run(); - await seedDeploymentIdentity(db, 'acme'); + await seedDeploymentIdentity(db, 'acme', 'open'); await expect(readDeploymentIdentity(db)).resolves.toBe('acme'); }); @@ -332,9 +424,9 @@ describe('deployment identity provisioning', () => { }, }; - await expect(seedDeploymentIdentity(racing, 'acme')).rejects.toThrow( - /raced_application/, - ); + await expect( + seedDeploymentIdentity(racing, 'acme', 'open'), + ).rejects.toThrow(/raced_application/); await expect(readDeploymentIdentity(base)).resolves.toBeUndefined(); }); }); @@ -342,7 +434,7 @@ describe('deployment identity provisioning', () => { describe('deployment identity runtime guard', () => { it('accepts a matching sentinel and fails closed on missing or mismatched identity', async () => { const matching = sqliteDatabase(); - await seedDeploymentIdentity(matching, 'acme'); + await seedDeploymentIdentity(matching, 'acme', 'open'); await expect(assertDeploymentIdentity(matching, 'acme')).resolves.toBe( undefined, ); @@ -360,7 +452,7 @@ describe('deployment identity runtime guard', () => { let reads = 0; let failures = 2; const seeded = sqliteDatabase(); - await seedDeploymentIdentity(seeded, 'acme'); + await seedDeploymentIdentity(seeded, 'acme', 'open'); const db = interceptSentinelRead(seeded, () => { reads += 1; if (failures > 0) { @@ -396,7 +488,7 @@ describe('deployment identity runtime guard', () => { it('requires matching bindings in production DOs and skips node-only instances', async () => { const db = sqliteDatabase(); - await seedDeploymentIdentity(db, 'acme'); + await seedDeploymentIdentity(db, 'acme', 'open'); const state = { id: { name: 'instance' } }; await expect( @@ -423,7 +515,7 @@ describe('deployment identity runtime guard', () => { it('validates all Worker bindings before touching routes', async () => { const db = sqliteDatabase(); - await seedDeploymentIdentity(db, 'acme'); + await seedDeploymentIdentity(db, 'acme', 'open'); await expect( ensureDeploymentIdentityBindings({ DB: db, @@ -448,8 +540,8 @@ describe('deployment identity runtime guard', () => { it('verifies every D1-shaped binding at Worker and Durable Object entry', async () => { const primary = sqliteDatabase(); const secondary = sqliteDatabase(); - await seedDeploymentIdentity(primary, 'acme'); - await seedDeploymentIdentity(secondary, 'other'); + await seedDeploymentIdentity(primary, 'acme', 'open'); + await seedDeploymentIdentity(secondary, 'other', 'open'); const env = { DB: primary, SCHEDULES_DB: secondary, @@ -465,7 +557,7 @@ describe('deployment identity runtime guard', () => { ).rejects.toThrow(/belongs to 'other'/); const matchingSecondary = sqliteDatabase(); - await seedDeploymentIdentity(matchingSecondary, 'acme'); + await seedDeploymentIdentity(matchingSecondary, 'acme', 'open'); const matchingEnv = { ...env, SCHEDULES_DB: matchingSecondary, @@ -477,7 +569,7 @@ describe('deployment identity runtime guard', () => { it('never adopts an RPC binding as a deployment database', async () => { const db = sqliteDatabase(); - await seedDeploymentIdentity(db, 'acme'); + await seedDeploymentIdentity(db, 'acme', 'open'); // A service binding with a named entrypoint, and a Durable Object stub, are // proxies that answer EVERY property with a callable. A `prepare`-only test // adopts them, and the sentinel scan then dies with "The RPC receiver does @@ -512,7 +604,7 @@ describe('deployment identity runtime guard', () => { // The positive control: a real second D1 binding is still scanned, so the // exclusion above cannot be over-broad. const secondary = sqliteDatabase(); - await seedDeploymentIdentity(secondary, 'other'); + await seedDeploymentIdentity(secondary, 'other', 'open'); const withSecondDatabase = { ...env, SCHEDULES_DB: secondary }; await expect( ensureDeploymentIdentityBindings(withSecondDatabase), @@ -521,7 +613,7 @@ describe('deployment identity runtime guard', () => { it('rejects a Worker request from a differently credentialed deployment', async () => { const db = sqliteDatabase(); - await seedDeploymentIdentity(db, 'acme'); + await seedDeploymentIdentity(db, 'acme', 'open'); const state = { id: { name: 'instance' } }; const env = { DB: db, @@ -557,7 +649,7 @@ describe('deployment identity runtime guard', () => { it('rejects the caller credential before reading any D1 sentinel', async () => { const seeded = sqliteDatabase(); - await seedDeploymentIdentity(seeded, 'acme'); + await seedDeploymentIdentity(seeded, 'acme', 'open'); let sentinelReads = 0; const db = interceptSentinelRead(seeded, () => { sentinelReads += 1; @@ -585,7 +677,7 @@ describe('deployment identity runtime guard', () => { it('rejects a malformed configured credential before reading D1', async () => { const seeded = sqliteDatabase(); - await seedDeploymentIdentity(seeded, 'acme'); + await seedDeploymentIdentity(seeded, 'acme', 'open'); let sentinelReads = 0; const db = interceptSentinelRead(seeded, () => { sentinelReads += 1; diff --git a/packages/flowsafe/src/do-runner/deployment-identity.ts b/packages/flowsafe/src/do-runner/deployment-identity.ts index af034aa..43301f4 100644 --- a/packages/flowsafe/src/do-runner/deployment-identity.ts +++ b/packages/flowsafe/src/do-runner/deployment-identity.ts @@ -24,6 +24,7 @@ import { DeploymentIdentityError, deploymentIdentityHeaders, type DeploymentIdentityProtocolExecutor, + type InitialExecutionFenceState, provisionDeploymentIdentityProtocol, readDeploymentIdentityProtocol, } from '#deployment-identity-protocol'; @@ -52,6 +53,13 @@ export { deploymentIdentityHeaders, }; +/** + * The execution-fence state a deployment is provisioned into — 'open' or + * 'migration-locked'. Named here so a host wiring `seedDeploymentIdentity` + * types the choice rather than passing a bare string. + */ +export type { InitialExecutionFenceState }; + /** * Minimal structural D1 surface the sentinel check uses — same posture as * SnapshotDatabase: tests back it with node:sqlite, Workers pass env.DB. @@ -84,7 +92,20 @@ function assertValidTag(tag: unknown, caller: string): asserts tag is string { assertValidDeploymentTag(tag, caller); } -function isDatabaseBinding(db: unknown): db is DeploymentIdentityDatabase { +/** + * Whether a binding is a D1 database rather than something else bound under the + * same name. + * + * Exported for this package's own modules and deliberately NOT re-exported from + * `./index.js` — the same posture `safeDecodeSegment` has. It is internal + * plumbing two files must agree on (durable-object.ts asks the same question of + * `env.DB` before it insists on a fence), not a contract a consumer should be + * able to pin; barrel-exporting it would inflate the semver surface of a + * predicate that exists to describe Cloudflare's binding shapes. + */ +export function isDatabaseBinding( + db: unknown, +): db is DeploymentIdentityDatabase { if ((typeof db !== 'object' && typeof db !== 'function') || db === null) { return false; } @@ -154,18 +175,26 @@ function deploymentIdentityProtocolExecutor( /** * Provisioning-time seeding: stamp the database with its owning deployment's - * tag. Idempotent for the SAME tag (re-running provisioning is safe); throws - * for a DIFFERENT tag — a database is never silently re-homed; deployment - * decommissioning deletes it instead. + * tag, and write its initial execution-fence row. Idempotent for the SAME tag + * (re-running provisioning is safe); throws for a DIFFERENT tag — a database is + * never silently re-homed; deployment decommissioning deletes it instead. + * + * `initialExecutionFenceState` is REQUIRED and has no default: a host that + * means to bring a deployment up already locked for a migration must say so, + * and one that forgets must not silently get an executing deployment. The fence + * row is INSERT-if-absent, so re-seeding never reopens a fence an operator + * closed. */ export async function seedDeploymentIdentity( db: DeploymentIdentityDatabase, tag: string, + initialExecutionFenceState: InitialExecutionFenceState, ): Promise { databaseBinding(db, 'seedDeploymentIdentity'); await provisionDeploymentIdentityProtocol( deploymentIdentityProtocolExecutor(db), tag, + { initialExecutionFenceState }, ); } diff --git a/packages/flowsafe/src/do-runner/do-error-response.ts b/packages/flowsafe/src/do-runner/do-error-response.ts index cb18876..d990ff0 100644 --- a/packages/flowsafe/src/do-runner/do-error-response.ts +++ b/packages/flowsafe/src/do-runner/do-error-response.ts @@ -13,6 +13,8 @@ // RunRouteError passthrough has nothing to pass through. import { DeploymentIdentityError } from './deployment-identity.js'; +import type { DoRefusalReason } from './do-status-error.js'; +import { DoStatusError } from './do-status-error.js'; import { InvalidRunRequestError, RunAlreadyExistsError, @@ -24,29 +26,11 @@ import { UnknownWorkflowError, } from './runtime.js'; -/** - * The base for a DO's OWN refusal — the taxonomy's extension point, for the - * statuses the runtime errors above do not cover. A shell declares one - * (ThreadIdentityError's 403) and so does a host route: - * - * ```ts - * class UnknownSignalError extends DoStatusError { - * readonly status = 404; - * } - * ``` - * - * A CLASS, so opting in is `instanceof` — deliberate and nominal, the posture - * AuditLogger takes. The structural alternative (any thrown - * value with a numeric `status`) cannot tell a refusal this DO authored from the - * arbitrary values its routes throw: an upstream client's `{status: 429}` would - * become this API's 429, and `{status: 0}` — routine on HTTP-client error - * objects — would make `new Response` raise inside the very catch whose job is - * to never throw. Everything unrecognized stays a 500. - */ -export abstract class DoStatusError extends Error { - /** The response status. 4xx/5xx only — see doErrorResponse. */ - abstract readonly status: number; -} +// Re-exported under the name every shell already imports; the class itself +// lives on a leaf so a refusal module can extend it without closing an import +// cycle through this mapper (see do-status-error.ts). +export type { DoRefusalReason }; +export { DoStatusError }; function statusOf(error: unknown): number | undefined { // Mis-provisioned deployment (env tag vs D1 sentinel): the operator's @@ -91,18 +75,31 @@ function statusOf(error: unknown): number | undefined { return undefined; } +/** + * The structured reason a refusal publishes, or undefined. Two channels, one + * renderer: RunLifecycleBlockedError predates DoStatusError and is a plain + * Error, while every refusal authored since carries its reason on the + * taxonomy's own base (ExecutionFencedError's EXECUTION_FENCED among them). + * Anything else has no reason to publish — an unclassified fault must not + * grow a machine-readable code it never defined. + */ +function reasonOf(error: unknown): unknown { + if (error instanceof RunLifecycleBlockedError) return error.reason; + if (error instanceof DoStatusError) return error.reason; + return undefined; +} + /** * Map a thrown error to the DO's HTTP response. Anything unrecognized is a 500 * with its message — the honest answer for a fault this layer did not classify. */ export function doErrorResponse(error: unknown): Response { const message = error instanceof Error ? error.message : String(error); + const reason = reasonOf(error); return new Response( JSON.stringify({ error: message, - ...(error instanceof RunLifecycleBlockedError - ? { reason: error.reason } - : {}), + ...(reason === undefined ? {} : { reason }), }), { status: statusOf(error) ?? 500, diff --git a/packages/flowsafe/src/do-runner/do-status-error.ts b/packages/flowsafe/src/do-runner/do-status-error.ts new file mode 100644 index 0000000..a8b0873 --- /dev/null +++ b/packages/flowsafe/src/do-runner/do-status-error.ts @@ -0,0 +1,51 @@ +// SPDX-License-Identifier: Apache-2.0 +// The taxonomy's extension point, on its own leaf. +// +// It lives apart from doErrorResponse (which imports it, and re-exports it as +// the public name) for one structural reason: every module that declares a +// refusal must `extends` this class at module-evaluation time, and +// doErrorResponse's own imports reach the runtime and the deployment identity +// guard. A refusal module importing the mapper would therefore close an import +// cycle whose failure mode is a temporal-dead-zone ReferenceError at +// `class X extends DoStatusError` — order-dependent, and invisible until some +// unrelated import order changes. A leaf with no imports of its own cannot be +// half-evaluated when a subclass needs it. + +/** A structured refusal code doErrorResponse renders into the body. */ +export interface DoRefusalReason { + /** SCREAMING_SNAKE, per the taxonomy — see do-error-response.ts. */ + readonly code: string; +} + +/** + * The base for a DO's OWN refusal — the taxonomy's extension point, for the + * statuses the runtime errors do not cover. A shell declares one + * (ThreadIdentityError's 403) and so does a host route: + * + * ```ts + * class UnknownSignalError extends DoStatusError { + * readonly status = 404; + * } + * ``` + * + * A CLASS, so opting in is `instanceof` — deliberate and nominal, the posture + * AuditLogger takes. The structural alternative (any thrown + * value with a numeric `status`) cannot tell a refusal this DO authored from the + * arbitrary values its routes throw: an upstream client's `{status: 429}` would + * become this API's 429, and `{status: 0}` — routine on HTTP-client error + * objects — would make `new Response` raise inside the very catch whose job is + * to never throw. Everything unrecognized stays a 500. + */ +export abstract class DoStatusError extends Error { + /** The response status. 4xx/5xx only — see doErrorResponse. */ + abstract readonly status: number; + + /** + * An optional machine-readable reason, rendered into the response body + * alongside the message. DECLARED HERE rather than sniffed structurally at + * the mapper so the channel is part of the contract a subclass opts into, + * and so the mapper needs no import of any subclass to render it (see the + * header for why that import must not exist). + */ + readonly reason?: DoRefusalReason; +} diff --git a/packages/flowsafe/src/do-runner/durable-object.test.ts b/packages/flowsafe/src/do-runner/durable-object.test.ts index 3ae4477..1622298 100644 --- a/packages/flowsafe/src/do-runner/durable-object.test.ts +++ b/packages/flowsafe/src/do-runner/durable-object.test.ts @@ -38,6 +38,8 @@ import { type DurableObjectRunOwnershipStore, nextDutyAlarmAt, } from './durable-object.js'; +import type { ExecutionFenceDatabase } from './execution-fence.js'; +import { ExecutionFenceStore } from './execution-fence.js'; import { EXECUTION_PRINCIPAL_HEADER } from './execution-principal-header.js'; import { init } from './init.js'; import { @@ -46,6 +48,8 @@ import { type RunSummary, } from './runtime.js'; import type { ScheduleSourceStore } from './schedule-source.js'; +import type { StartIdempotencyDatabase } from './start-idempotency.js'; +import { StartIdempotencyStore } from './start-idempotency.js'; import { isSuspensionTimeoutResumeData, MAX_SUSPENSION_DEADLINE_ATTEMPTS, @@ -62,6 +66,13 @@ import { interface TestEnv extends DeploymentIdentityEnv { storage: InMemoryStore; runtime?: RunnerRuntime; + /** + * The deployment execution fence the built runtime is wired to. Always + * present on a production-shaped env — DurableObjectRunner refuses to serve + * from a fence-less RunnerRuntime while a DB binding is bound — and shared + * with the test so it can move the fence under a live object. + */ + fence?: ExecutionFenceStore; owners: DurableObjectRunOwnershipStore; schedules?: ScheduleSourceStore; lifecycle?: DurableObjectRunLifecycleHooks; @@ -110,12 +121,17 @@ function makeProductionEnv( }, } : registry; + const db = deploymentIdentityDatabase(); return { storage, owners, DEPLOYMENT_TENANT: 'acme', DEPLOYMENT_IDENTITY_SECRET: TEST_DEPLOYMENT_IDENTITY_SECRET, - DB: deploymentIdentityDatabase(), + DB: db, + // The fence lives in the SAME database as the deployment sentinel, exactly + // as it does in production. Absent-table reads as 'open', so every test + // that does not move it is byte-identical to before the fence existed. + fence: new ExecutionFenceStore(db), }; } @@ -133,8 +149,40 @@ function statusStub(read: RunnerRuntime['status']) { return { status: vi.fn(read), authoritativeStatus: vi.fn(read) }; } -function gatedRuntime(storage: InMemoryStore): RunnerRuntime { - const { createWorkflow, createStep, runtime } = init({ storage }); +/** + * A fence over its own throwaway database, for the runtime builders that are + * handed a storage instance and no env. Every read finds no table and answers + * 'open', so these runners behave exactly as they did before the fence — what + * it buys is that they are FENCED runtimes, which is what DurableObjectRunner + * asserts of anything it serves from while a DB binding is bound. + */ +function newTestExecutionFence(): ExecutionFenceStore { + return new ExecutionFenceStore( + sqliteUnitDatabase(openSqlite()) as ExecutionFenceDatabase, + ); +} + +/** + * A start-reservation store over its own throwaway database, for the same + * reason as the fence above: DurableObjectRunner refuses to serve from a + * runtime that has none while a DB binding is bound, and every runner in this + * file carries one. No key is ever used against it, so the table is never even + * created and every runner behaves exactly as it did before reservations. + */ +function newTestStartIdempotency(): StartIdempotencyStore { + return new StartIdempotencyStore( + sqliteUnitDatabase(openSqlite()) as StartIdempotencyDatabase, + ); +} + +function gatedRuntime( + storage: InMemoryStore, + executionFence: ExecutionFenceStore = newTestExecutionFence(), +): RunnerRuntime { + const { createWorkflow, createStep, runtime } = init( + { storage }, + { executionFence, startIdempotency: newTestStartIdempotency() }, + ); const gate = createStep({ id: 'gate', inputSchema: z.object({ topic: z.string() }), @@ -171,7 +219,7 @@ class TestRunner extends DurableObjectRunner { protected build(env: TestEnv): RunnerRuntime { if (env.runtime) return env.runtime; - return gatedRuntime(env.storage); + return gatedRuntime(env.storage, env.fence); } } @@ -1102,7 +1150,9 @@ describe('DurableObjectRunner.fetch', () => { const approvalStore = new InMemoryApprovalStore(); const filed = await reconcileApprovalsForSummary( - new ApprovalService({ store: approvalStore }), + // In-memory store, no database to fence against: the opt-out is written down + // rather than defaulted — see ExecutionFenceWiring. + new ApprovalService({ store: approvalStore, executionFence: 'none' }), 'gated', summary, 'approval-reconciler', @@ -1384,7 +1434,7 @@ describe('DurableObjectRunner.fetch', () => { } }); - it('400s a start without a runId — the DO never generates one (INV-1)', async () => { + it('400s a start without a runId — the DO never generates one', async () => { // #given — the runId is host-owned and minted by the run router. A DO-side // generation fallback would let any caller that skips the router create an // unowned run outside the request budget and audit boundary. @@ -1400,7 +1450,7 @@ describe('DurableObjectRunner.fetch', () => { } }); - it('refuses to act outside its own identity when id.name is present (INV-1)', async () => { + it('refuses to act outside its own identity when id.name is present', async () => { // #given — a runner whose DO identity names a DIFFERENT run than the // request. id.name is set by the trusted Worker via idFromName and is // unforgeable at this boundary, so a mismatch means someone routed @@ -1458,9 +1508,13 @@ describe('DurableObjectRunner.fetch', () => { } protected build(env: TestEnv): RunnerRuntime { - const { createWorkflow, createStep, runtime } = init({ - storage: env.storage, - }); + const { createWorkflow, createStep, runtime } = init( + { storage: env.storage }, + { + executionFence: env.fence ?? newTestExecutionFence(), + startIdempotency: newTestStartIdempotency(), + }, + ); const gate = createStep({ id: 'gate', inputSchema: z.object({}), @@ -1701,7 +1755,7 @@ describe('DurableObjectRunner.fetch', () => { let wedge = true; const env = makeProductionEnv(storage); env.runtime = new Proxy(runtime, { - get(target, property, receiver) { + get(target, property) { if (property === 'completeTerminalCleanup') { return async ( ...args: Parameters @@ -1713,7 +1767,11 @@ describe('DurableObjectRunner.fetch', () => { return complete(...args); }; } - const value = Reflect.get(target, property, receiver) as unknown; + // Receiver is the TARGET, not the proxy: RunnerRuntime's accessors read + // private fields, and a private-field read against a proxy receiver + // throws. Every function is re-bound to the target below for the same + // reason, so this only makes the getters agree with the methods. + const value = Reflect.get(target, property, target) as unknown; return typeof value === 'function' ? value.bind(target) : value; }, }); @@ -1915,8 +1973,8 @@ describe('DurableObjectRunner.fetch', () => { it('returns a 426 non-WS fallback on the stream route when the runtime has no hibernation API', async () => { // #given — a node runner (state undefined ⇒ no acceptWebSocket). The per-run - // WS stream is workerd-only; off workerd it must degrade, never 500. The WS - // runtime behavior itself is proven by the workerd spike (M-009). + // WS stream is workerd-only; off workerd it must degrade, never 500. The + // WebSocket runtime behavior itself is proven by the workerd spike. const runner = makeRunner(); // #when — a websocket upgrade attempt on the stream route @@ -1930,7 +1988,7 @@ describe('DurableObjectRunner.fetch', () => { expect(response.status).toBe(426); }); - it('broadcasts the authoritative RunSummary to run-channel sockets after start and resume (DL-018)', async () => { + it('broadcasts the authoritative RunSummary to run-channel sockets after start and resume', async () => { // #given — a runner whose DO exposes a hibernatable-socket stub; // #broadcastRunSummary reads getWebSockets() and send()s each the frame. const sent: string[] = []; @@ -2004,8 +2062,12 @@ type TimedStepExecute = ExecuteFunction< function timedRuntime( storage: InMemoryStore, onSettle?: () => void, + executionFence: ExecutionFenceStore = newTestExecutionFence(), ): RunnerRuntime { - const { createWorkflow, createStep, runtime } = init({ storage }); + const { createWorkflow, createStep, runtime } = init( + { storage }, + { executionFence, startIdempotency: newTestStartIdempotency() }, + ); const timedStep = (id: string, execute: TimedStepExecute) => createStep({ id, @@ -2113,7 +2175,13 @@ function collidingRuntime(storage: InMemoryStore): { settled: () => string[]; } { const settled: string[] = []; - const { createWorkflow, createStep, runtime } = init({ storage }); + const { createWorkflow, createStep, runtime } = init( + { storage }, + { + executionFence: newTestExecutionFence(), + startIdempotency: newTestStartIdempotency(), + }, + ); const suspending = (id: string, label: string) => createStep({ id, @@ -2157,7 +2225,13 @@ function foreachRuntime( options?: { concurrency: number }, ): { runtime: RunnerRuntime; timedOut: () => number[] } { const timedOut: number[] = []; - const { createWorkflow, createStep, runtime } = init({ storage }); + const { createWorkflow, createStep, runtime } = init( + { storage }, + { + executionFence: newTestExecutionFence(), + startIdempotency: newTestStartIdempotency(), + }, + ); const gate = createStep({ id: 'gate', inputSchema: z.object({ item: z.number() }), @@ -2193,7 +2267,7 @@ function foreachRuntime( function timedEnv(): TestEnv { const env = makeProductionEnv(); - env.runtime = timedRuntime(env.storage); + env.runtime = timedRuntime(env.storage, undefined, env.fence); return env; } @@ -5302,3 +5376,436 @@ describe('nextDutyAlarmAt', () => { expect(nextDutyAlarmAt(NOW - 5_000, NOW + 60_000, NOW)).toBe(NOW + 1_000); }); }); + +describe('DurableObjectRunner and the deployment execution fence', () => { + it('refuses a fenced start before ANY of its own storage writes', async () => { + // #given — a locked deployment and a start that would otherwise journal a + // recovery record, arm an alarm, and reserve the run's owner. + const events: string[] = []; + const { state } = recoveryStorage(events); + const reserve = vi.fn(async () => true); + const env = makeProductionEnv(new InMemoryStore(), { + reserve, + settle: vi.fn(async () => undefined), + }); + await env.fence?.seed('migration-locked'); + const runner = new TestRunner(state, env); + events.length = 0; + + // #when + const response = await runner.fetch( + post('/runs', { + workflowId: 'gated', + runId: 'fenced-start', + inputData: { topic: 't' }, + }), + ); + + // #then — the refusal carries the taxonomy's retryable status and code. + expect(response.status).toBe(503); + expect(await response.json()).toEqual({ + error: + "deployment execution is fenced ('migration-locked'): run start is refused", + reason: { code: 'EXECUTION_FENCED', state: 'migration-locked' }, + }); + + // #then — and NOTHING was written on the way to saying no: no owner + // reservation in D1, and no DO-storage mutation at all (the recovery + // journal, its alarm, or any delete). A deployment whose state is being + // copied must not leave a run half-claimed behind the copy. + expect(reserve).not.toHaveBeenCalled(); + expect( + events.filter( + (event) => + event.startsWith('put:') || + event.startsWith('delete:') || + event === 'setAlarm' || + event === 'deleteAlarm', + ), + ).toEqual([]); + }); + + it('keeps reads open while locked', async () => { + // #given — a run started before the lock. + const env = timedEnv(); + const { state } = recoveryStorage(); + const runner = new TestRunner(state, env); + await startTimed(runner, 'fenced-read'); + await env.fence?.seed('open'); + await env.fence?.transition({ + expected: 'open', + next: 'migration-locked', + }); + + // #when / #then — status still answers. An operator proving a deployment + // drained needs to read it, and a read moves nothing. + const response = await runner.fetch( + deploymentIdentityRequest('http://do/runs/timed/fenced-read'), + ); + expect(response.status).toBe(200); + expect((await response.json()) as RunSummary).toMatchObject({ + runId: 'fenced-read', + status: 'suspended', + }); + }); + + it('leaves a due deadline uncharged and unconverged under a locked fence, then fires it after reopen', async () => { + // #given — a suspended run with a due deadline on a locked deployment. + const env = timedEnv(); + const { state, values, alarms } = recoveryStorage(); + const runner = new TestRunner(state, env); + await startTimed(runner, 'fenced-deadline'); + elapseDeadlines(values); + const armed = storedEntry(values, 'gate'); + await env.fence?.seed('open'); + await env.fence?.transition({ + expected: 'open', + next: 'migration-locked', + }); + // Only the wakes below are under test; the start's own arm is not. + alarms.length = 0; + const logged: string[] = []; + const log = vi + .spyOn(console, 'error') + .mockImplementation((...args: unknown[]) => { + logged.push(String(args[0])); + }); + const before = Date.now(); + + // #when — three wakes under the lock. + try { + for (let wake = 0; wake < 3; wake += 1) { + // #then — an alarm NEVER rethrows: workerd retries a thrown alarm up + // to six times, which would answer a deliberate operational state with + // a wake storm. + await expect(runner.alarm()).resolves.toBeUndefined(); + } + } finally { + log.mockRestore(); + } + + // #then — every arm is the 60 s watchdog, never the floored re-arm a + // still-due entry would produce. The wake converged nothing, so it has no + // arm of its own to compute. + expect(alarms.filter((at) => at <= before)).toEqual([]); + for (const at of alarms) { + expect(at).toBeGreaterThanOrEqual(before + 60_000); + } + // #then — and the entry is untouched: not charged (five charged wakes + // would tombstone it in about sixteen minutes of lock), not tombstoned, + // and not stamped with the unreadable clock either — the read SUCCEEDED, + // the deployment simply refused, and that clock's day-long abandonment + // budget exists for a run whose state is permanently unreadable. + const entry = storedEntry(values, 'gate'); + expect(entry).toEqual(armed); + expect(entry).not.toHaveProperty('attempts'); + expect(entry).not.toHaveProperty('nextAttemptAt'); + expect(entry).not.toHaveProperty('unreadableSince'); + expect( + logged.filter((message) => + message.includes('refused by the deployment execution fence'), + ), + ).toHaveLength(3); + expect(logged).not.toContain('suspension deadline wake failed'); + + // #when — the migration finishes and the operator reopens the fence. + await env.fence?.transition({ expected: 'migration-locked', next: 'open' }); + await runner.alarm(); + + // #then — the deadline fires. Nothing was lost while the fence was closed. + const settled = await runner.fetch( + deploymentIdentityRequest('http://do/runs/timed/fenced-deadline'), + ); + expect((await settled.json()) as RunSummary).toMatchObject({ + status: 'success', + result: { settledBy: 'timeout' }, + }); + }); + + it('refuses to serve from a fence-less runtime while a database is bound', async () => { + // #given — a host that built a RunnerRuntime by hand inside build() and + // forgot the fence, on a deployment that HAS a database. + const env = makeProductionEnv(); + env.runtime = init( + { storage: env.storage }, + { startIdempotency: 'none', executionFence: 'none' }, + ).runtime; + const runner = new TestRunner(undefined, env); + + // #then — refused at the first request rather than silently executing + // straight through a migration lock. Every other surface would report the + // fence as wired, so nothing else would catch this. The MESSAGE is pinned + // too: a bare 500 could be any fault, and this test would still pass while + // the guard it exists for had stopped firing. + const response = await runner.fetch( + post('/runs', { workflowId: 'gated', runId: 'no-fence' }), + ); + expect(response.status).toBe(500); + expect(await response.json()).toMatchObject({ + error: expect.stringContaining( + 'returned a runtime with no execution fence', + ), + }); + }); + + it('serves from a fence-less runtime when DB is an RPC binding, not a database', async () => { + // #given — the OTHER direction of the same discrimination, and the one that + // fails loudly for every host if it is wrong. `DB` here is a service + // binding with a named entrypoint (fleet trusted state binds exactly this + // beside its D1), which is an RPC proxy: it answers EVERY property with a + // callable, so a bare `prepare` test says yes to it. There is no database + // to fence against, so a fence-less runtime is correct — insisting on a + // fence would refuse the first request every such Worker ever serves. + const env = makeProductionEnv(); + env.DB = new Proxy( + {}, + { get: () => () => undefined }, + ) as unknown as typeof env.DB; + env.runtime = init( + { storage: env.storage }, + { startIdempotency: 'none', executionFence: 'none' }, + ).runtime; + const runner = new TestRunner(undefined, env); + + // #then — past the guard. 404 is this bare runtime answering for a workflow + // it was never given; what matters is that it ANSWERED, where the D1-shaped + // binding above produced the guard's 500. + const response = await runner.fetch( + post('/runs', { workflowId: 'gated', runId: 'rpc-db' }), + ); + expect(response.status).toBe(404); + }); +}); + +// --------------------------------------------------------------------------- +// What the run object contributes to an idempotent start: it carries the +// key on the internal channel, and it answers the liveness probe that separates +// a run still working from a claim nobody is holding. +// --------------------------------------------------------------------------- + +describe('DurableObjectRunner — idempotent start plumbing', () => { + it('answers the liveness probe false for a run it is not executing', async () => { + // #given a run object with nothing in flight + const runner = makeRunner(); + + // #when + const response = await runner.fetch( + deploymentIdentityRequest('http://do/runs/gated/run-idle/start-liveness'), + ); + + // #then. `false` is the fail-closed direction here: it produces the + // refusal that asks a human to investigate, where a default of `true` + // would answer a permanently dead run with a permanently retryable 503. + expect(response.status).toBe(200); + expect(await response.json()).toEqual({ live: false }); + }); + + it('answers the liveness probe true WHILE the start is executing, without queuing behind it', async () => { + // #given a workflow whose first step blocks until the probe has answered. + // This is the whole point of the route: the start holds the operation lock + // for its entire first leg, so a probe that took that lock would block for + // exactly as long as the run it was trying to describe. + const storage = new InMemoryStore(); + const { createWorkflow, createStep, runtime } = init( + { storage }, + { + executionFence: newTestExecutionFence(), + startIdempotency: newTestStartIdempotency(), + }, + ); + let probed!: (value: unknown) => void; + const probeAnswered = new Promise((resolve) => { + probed = resolve; + }); + let running!: (value: unknown) => void; + const stepRunning = new Promise((resolve) => { + running = resolve; + }); + const blocking = createStep({ + id: 'blocking', + inputSchema: z.object({}), + outputSchema: z.object({}), + execute: async () => { + running(undefined); + await probeAnswered; + return {}; + }, + }); + createWorkflow({ + id: 'gated', + inputSchema: z.object({}), + outputSchema: z.object({}), + }) + .then(blocking) + .commit(); + const env = { ...makeProductionEnv(storage), runtime }; + const runner = new TestRunner(undefined, env); + + // #when the start is in flight and a probe arrives + const started = runner.fetch( + post('/runs', { workflowId: 'gated', runId: 'run-live', inputData: {} }), + ); + await stepRunning; + const probe = await runner.fetch( + deploymentIdentityRequest('http://do/runs/gated/run-live/start-liveness'), + ); + probed(undefined); + await started; + + // #then the probe answered — promptly, and truthfully + expect(await probe.json()).toEqual({ live: true }); + + // #and once the start is done, so is the liveness + const after = await runner.fetch( + deploymentIdentityRequest('http://do/runs/gated/run-live/start-liveness'), + ); + expect(await after.json()).toEqual({ live: false }); + }); + + it('refuses an idempotency key that is not path-safe', async () => { + // #given — the same string is compared against the fence's proof key and + // stored as a reservation's primary key, so an unvalidated one reaches both + const runner = makeRunner(); + + // #when + const response = await runner.fetch( + post('/runs', { + workflowId: 'gated', + runId: 'run-bad-key', + inputData: { topic: 't' }, + idempotencyKey: 'key/../escape', + }), + ); + + // #then + expect(response.status).toBe(400); + expect(await response.json()).toMatchObject({ + error: expect.stringContaining('idempotencyKey'), + }); + }); + + it('admits exactly the proof-only start that carries the nominated key, end to end', async () => { + // #given a deployment fenced into proof-only, addressed through the route + // a trusted Worker actually uses + const storage = new InMemoryStore(); + const env = makeProductionEnv(storage); + const fence = env.fence as ExecutionFenceStore; + await fence.seed('migration-locked'); + await fence.transition({ + expected: 'migration-locked', + next: 'proof-only', + proofKey: 'proof-key-1', + }); + const runner = new TestRunner(undefined, env); + + // #then a start with NO key is refused at the route, before this object + // writes anything of its own + const unkeyed = await runner.fetch( + post('/runs', { + workflowId: 'gated', + runId: 'run-unkeyed', + inputData: { topic: 't' }, + }), + ); + expect(unkeyed.status).toBe(503); + expect(await unkeyed.json()).toMatchObject({ + reason: { code: 'EXECUTION_FENCED', state: 'proof-only' }, + }); + + // #and a start carrying the WRONG key is refused the same way + const guessed = await runner.fetch( + post('/runs', { + workflowId: 'gated', + runId: 'run-guessed', + inputData: { topic: 't' }, + idempotencyKey: 'guessed-key', + }), + ); + expect(guessed.status).toBe(503); + + // #and the nominated start is admitted AND binds the proof run + const admitted = await runner.fetch( + post('/runs', { + workflowId: 'gated', + runId: 'run-proof', + inputData: { topic: 't' }, + idempotencyKey: 'proof-key-1', + }), + ); + expect(admitted.status).toBe(200); + await expect(fence.read()).resolves.toEqual({ + state: 'proof-only', + proofKey: 'proof-key-1', + proofRunId: 'run-proof', + }); + + // #and a SECOND start under the same key is refused: the proof is one run, + // and recordProofRun's CAS is what says so. + const second = await runner.fetch( + post('/runs', { + workflowId: 'gated', + runId: 'run-proof-2', + inputData: { topic: 't' }, + idempotencyKey: 'proof-key-1', + }), + ); + expect(second.status).toBe(503); + }); + + it('refuses an admitted start whose fence MOVED before the write-back landed', async () => { + // #given a fence that reads proof-only and then, between the admitting + // read and the write-back, has been transitioned away — the 0-row case + // recordProofRun's CAS exists for + const storage = new InMemoryStore(); + const env = makeProductionEnv(storage); + const fence = env.fence as ExecutionFenceStore; + await fence.seed('migration-locked'); + await fence.transition({ + expected: 'migration-locked', + next: 'proof-only', + proofKey: 'proof-key-1', + }); + let moved = false; + const moving = new Proxy(fence, { + get(target, property, receiver) { + if (property === 'read') { + return async () => { + const reading = await target.read(); + // The operator moves the fence ONCE, right after the read that + // admitted the start. Every later read sees the moved fence, which + // is exactly what an admitted-then-moved start observes. + if (!moved) { + moved = true; + await target.transition({ + expected: 'proof-only', + next: 'migration-locked', + }); + } + return reading; + }; + } + const value = Reflect.get(target, property, receiver) as unknown; + return typeof value === 'function' ? value.bind(target) : value; + }, + }); + const runner = new TestRunner(undefined, { + ...env, + fence: moving as ExecutionFenceStore, + }); + + // #when + const response = await runner.fetch( + post('/runs', { + workflowId: 'gated', + runId: 'run-proof', + inputData: { topic: 't' }, + idempotencyKey: 'proof-key-1', + }), + ); + + // #then refused, and nothing ran: the deployment is no longer the one this + // start read, so its admission is void. + expect(response.status).toBe(503); + await expect(fence.read()).resolves.toEqual({ state: 'migration-locked' }); + }); +}); diff --git a/packages/flowsafe/src/do-runner/durable-object.ts b/packages/flowsafe/src/do-runner/durable-object.ts index f50ea08..3ea9ee7 100644 --- a/packages/flowsafe/src/do-runner/durable-object.ts +++ b/packages/flowsafe/src/do-runner/durable-object.ts @@ -22,10 +22,19 @@ import { import type { DurableObjectRunnerState, WebSocketLike } from './cf-types.js'; import { newWebSocketPair, safeSend } from './cf-types.js'; import { + isDatabaseBinding, verifyDurableObjectDeploymentIdentity, verifyDurableObjectDeploymentRequest, } from './deployment-identity.js'; import { DoStatusError, doErrorResponse } from './do-error-response.js'; +import { + admitsExistingRun, + admitsRunStart, + ExecutionFencedError, + type ExecutionFenceReading, + isExecutionFenceRefusal, + readExecutionFence, +} from './execution-fence.js'; import { EXECUTION_PRINCIPAL_HEADER } from './execution-principal-header.js'; import { isPathSafeId } from './path-safe-id.js'; import { @@ -33,7 +42,7 @@ import { RunAlreadyExistsError, type RunLifecycleCas, type RunLifecycleTransitionResult, - type RunnerRuntime, + RunnerRuntime, RunStateUnreadableError, type RunSummary, UnknownRunError, @@ -103,7 +112,7 @@ export interface DurableObjectRunLifecycleHooks { } const RUN_OWNER_RECOVERY_KEY = 'flowsafe:run-owner-recovery:v1'; -const RUN_OWNER_RECOVERY_DELAY_MS = 60_000; +export const RUN_OWNER_RECOVERY_DELAY_MS = 60_000; const SUSPENSION_DEADLINE_RETRY_MS = 60_000; // How long a run's state may stay unreadable before the entries due under it // are given up on. A read that never succeeds is never charged, so without this @@ -151,6 +160,16 @@ interface StartBody { scheduleId?: unknown; dispatchId?: unknown; deadlineMs?: unknown; + /** + * The start's idempotency key, arriving on the INTERNAL Worker-to-DO channel + * only. Every request that reaches this route carries the deployment-identity + * header (a tenant request bearing one is refused before routing), so a key + * here is one the trusted run router reserved — which is what lets the + * proof-only fence match it. It is deliberately not an open request-context + * key and never a field of the public POST /runs body. + * @internal + */ + idempotencyKey?: unknown; } interface ResumeBody { @@ -170,6 +189,55 @@ class DurableObjectRunIdentityError extends DoStatusError { readonly status = 403; } +/** + * Does this env carry a database to fence against? + * + * `isDatabaseBinding` is deployment-identity's, imported rather than restated: + * it is the SAME question about the SAME binding (an RPC binding — a service + * binding with a named entrypoint, a Durable Object stub — is a proxy that + * answers every property with a callable, so `fetch` is what separates that + * family from a D1Database). Two copies would be two places for that + * discrimination to be revised, and a deployment where they disagreed would + * pass the identity check and skip the fence assert, or the reverse. + */ +function hasDatabaseBinding(env: unknown): boolean { + return isDatabaseBinding((env as { DB?: unknown } | null | undefined)?.DB); +} + +/** + * A run object bound to a real database must never serve from a fence-less + * runtime. `init()` makes that true for every host that hands it the binding — + * its `{ DB }` branch has no opt-out — so what is left for this assert is the + * host that builds a RunnerRuntime by hand inside `build()` and forgets: that + * deployment would then execute straight through a migration lock, silently, + * and the fence would look wired because every OTHER surface reports it. + * + * Scoped to runtimes THIS PACKAGE built. A test double cast to the type is not + * a RunnerRuntime and carries no fence by construction; asserting on it would + * indict every stub for a property it was never meant to have, and would say + * nothing about the production wiring this guard exists for. + */ +function assertFencedRuntime(runtime: RunnerRuntime, env: unknown): void { + if (!(runtime instanceof RunnerRuntime)) return; + if (!hasDatabaseBinding(env)) return; + if (runtime.executionFence === undefined) { + throw new Error( + 'DurableObjectRunner: build() returned a runtime with no execution fence while this deployment carries a DB binding — wire it through init({ DB }) (which builds one) or pass an ExecutionFenceStore, so a migration-locked deployment cannot execute', + ); + } + // The same guard, for the same failure. A hand-built runtime with no + // reservation store still EXECUTES idempotent starts — the router reserves + // and claims above it — but never marks their reservations spent, so every + // key this deployment ever honoured stays in the drain inventory and never + // becomes purgeable. That is invisible until the day someone tries to prove + // the deployment empty, which is the day it matters most. + if (runtime.startIdempotency === undefined) { + throw new Error( + 'DurableObjectRunner: build() returned a runtime with no start-reservation store while this deployment carries a DB binding — wire it through init({ DB }) (which builds one), so an idempotent start can be marked settled when its run ends', + ); + } +} + export abstract class DurableObjectRunner { protected readonly env: TEnv; /** Absent in Node tests; present under workerd for storage and the alarm. */ @@ -178,6 +246,16 @@ export abstract class DurableObjectRunner { #operationTail = Promise.resolve(); /** `step\0reason` of every suspension deadline this object has reported. */ #reportedSuspensionRejections = new Set(); + /** + * `workflowId:runId` of every start this object is currently executing — the + * liveness half of the idempotent-start replay decision. + * + * A SET rather than a stored key, because liveness is not durable state: the + * question is "is code running for this run right now", and the honest answer + * after an eviction is no. Anything written to storage would survive the + * isolate that wrote it and keep saying yes. + */ + readonly #startsInFlight = new Set(); constructor(state: DurableObjectRunnerState | undefined, env: TEnv) { this.state = state; @@ -218,13 +296,25 @@ export abstract class DurableObjectRunner { #ensureRuntime(): RunnerRuntime { if (!this.#runtime) { - this.#runtime = this.build(this.env); + const runtime = this.build(this.env); + assertFencedRuntime(runtime, this.env); + this.#runtime = runtime; } return this.#runtime; } - // INV-1 enforcement at the DO boundary: this instance was addressed as - // idFromName(`${workflowId}:${runId}`) by the trusted Worker, and id.name + /** + * This object's execution fence, or the open reading when the host built an + * unfenced runtime. Taken from the RUNTIME rather than constructed here, so + * the routes below and the start/resume backstop inside the runtime can + * never be gating two different databases. + */ + async #readExecutionFence(): Promise { + return readExecutionFence(this.#ensureRuntime().executionFence); + } + + // Enforce host-owned run ids at the DO boundary: this instance was addressed + // as idFromName(`${workflowId}:${runId}`) by the trusted Worker, and id.name // is unforgeable at this boundary. If a request asks the instance to act on // a DIFFERENT (workflowId, runId), someone routed around the name join — // acting on the request's ids would run outside this instance's identity @@ -237,11 +327,53 @@ export abstract class DurableObjectRunner { if (name === undefined) return; if (name !== `${workflowId}:${runId}`) { throw new Error( - `DO identity mismatch: instance is '${name}' but the request names '${workflowId}:${runId}' — refusing (INV-1)`, + `DO identity mismatch: instance is '${name}' but the request names '${workflowId}:${runId}' — refusing`, ); } } + /** + * The same `workflowId:runId` join the DO name and the runtime's own run key + * use — composed here so the in-flight set cannot be keyed by runId alone, + * which would make two workflows' runs of the same id one another's liveness. + */ + #inFlightKey(workflowId: string, runId: string): string { + return `${workflowId}:${runId}`; + } + + /** + * Is a start for this run executing in THIS object right now? + * + * Two sources, ORed, because they cover different halves of the window: the + * route's own set covers from the recovery journal to the response (including + * the gap before core persists anything), and the runtime's `#activeRuns` + * covers a run driven through this isolate's runtime by any other path — an + * in-process host, a resume, an agent loop sharing the runtime. Neither alone + * is the whole window, and a false negative here is the expensive direction: + * it turns a live run into an UNRESOLVABLE refusal. + */ + #isStartLive(workflowId: string, runId: string): boolean { + if (this.#startsInFlight.has(this.#inFlightKey(workflowId, runId))) { + return true; + } + return this.#ensureRuntime().isRunActive(workflowId, runId); + } + + /** + * Validate an internal-channel idempotency key. Absent stays absent; anything + * present must be path-safe, because the same string is compared against the + * fence's proof key and stored as a reservation's primary key. + */ + #startIdempotencyKey(value: unknown): string | undefined { + if (value === undefined || value === null) return undefined; + if (!isPathSafeId(value)) { + throw new InvalidRunRequestError( + 'idempotencyKey must be a URL-path-safe identifier', + ); + } + return value; + } + #requestedBy(value: unknown): string { if (!isExecutionPrincipalId(value)) { throw new InvalidRunRequestError('run requester is missing or malformed'); @@ -937,11 +1069,11 @@ export abstract class DurableObjectRunner { now: number, ): Promise { const { workflowId, runId } = stored; - // INV-1 at the one boundary whose ids come from storage rather than from - // the trusted Worker: this wake is about to execute a workflow body, so it - // must be this object's own run (and its per-run serialization). Charged - // like any other duty failure, deliberately: it is the one charged failure - // that read nothing, and charging is what quiets a foreign record's entry + // Enforce host-owned run ids at the boundary whose ids come from storage, + // rather than from the trusted Worker. This wake is about to execute a + // workflow body, so it must be this object's own run and serialization. + // Charge it like any other duty failure: it is the one charged failure that + // read nothing, and charging is what quiets a foreign record's entry // (five wakes to a tombstone). It stays here rather than moving up with the // runtime because the nothing-due path below deliberately prefers `id.name` // over a foreign record, which asserting early would break. @@ -1156,6 +1288,28 @@ export abstract class DurableObjectRunner { await this.#resumeDueSuspensionDeadline(runtime, stored, entry, now); return true; } catch (error) { + // The deployment is fenced (or its fence could not be read). Classified + // with the same THREE outcomes as an unreadable read — uncharged, + // unconverged, watchdog cadence — because a wake refused by an + // operational control is no evidence about the entry it was working on. + // Charging it would spend the abandonment budget in about sixteen + // minutes of lock and tombstone a live deadline; converging would arm + // from a still-due entry no ledger backed off, which lands on the + // one-second floor and spins. + // + // What it deliberately does NOT do is stamp `unreadableSince`. That + // clock exists to bound a run whose state became PERMANENTLY unreadable, + // and abandons its entries after a day; a fence is a deliberate, + // operator-visible, bounded state, and answering a long migration by + // discarding every deadline due inside it would be the same fault the + // no-charge rule above exists to prevent, one order of magnitude later. + if (isExecutionFenceRefusal(error)) { + console.error( + 'suspension deadline wake refused by the deployment execution fence', + error, + ); + return false; + } // Classified FIRST, or every degraded wake would also log the generic // failure below. A read that did not succeed converges nothing and // charges nothing: it keeps the watchdog cadence until it heals, and the @@ -1316,9 +1470,9 @@ export abstract class DurableObjectRunner { if (!body || typeof body.workflowId !== 'string') { return json({ error: 'workflowId is required' }, 400); } - // The DO never generates a runId (INV-1): the trusted Worker mints the - // id and addresses this instance with it. A start without one is a - // caller bug, not a request for generation. + // The DO never generates a runId: the trusted Worker mints the id and + // addresses this instance with it. A start without one is a caller bug, + // not a request for generation. if (typeof body.runId !== 'string') { return json( { error: 'runId is required (server-minted by the run router)' }, @@ -1333,6 +1487,30 @@ export abstract class DurableObjectRunner { ); } this.#assertRunIdentity(workflowId, runId); + // The key rides the internal channel and nothing else. Validated here + // rather than trusted because this body is JSON: an unvalidated value + // would reach the fence's proof-only comparison and the runtime's + // reservation as whatever the parser produced. + const idempotencyKey = this.#startIdempotencyKey(body.idempotencyKey); + // The fence BEFORE any of this object's own reads or writes: the + // schedule-source lookup below, the recovery pass, the journal at + // #armRunOwnerRecovery, and the owner reservation all touch storage, + // and a deployment that is refusing to execute must not leave a run + // half-claimed on its way to saying no. The runtime's own check inside + // start() stays the backstop for every other caller. + // + // The KEY is what admits a proof-only start: in that state the fence + // nominates exactly one key, and a start carrying it is the proof run. + // This check reads the fence but does NOT bind the proof to the run — + // `recordProofRun` belongs to the runtime's own assert, which is the + // last gate before execution and the only one every caller passes. + // Binding here as well would let a start that this route later refused + // (an existing run, a schedule-source mismatch) consume the deployment's + // one proof slot. + const startFence = await this.#readExecutionFence(); + if (!admitsRunStart(startFence, idempotencyKey)) { + throw new ExecutionFencedError(startFence.state, 'run start'); + } const source = await this.#startSource( principal, workflowId, @@ -1371,6 +1549,16 @@ export abstract class DurableObjectRunner { token: crypto.randomUUID(), }; await this.#armRunOwnerRecovery(recovery); + // From here to the finally below, this object IS the run's execution: + // everything past the journal either persists a snapshot or leaves the + // recovery pass to settle it. That window is exactly what a replaying + // start's liveness probe is asking about, and it is tracked in memory + // on purpose — an evicted isolate loses the entry, which is the true + // answer for a run that is no longer executing anywhere. Registered + // BEFORE the reservation and the runtime's own #activeRuns entry so the + // gap between the claim and core's first persisted snapshot — the one + // window where nothing else can see the run — is covered too. + this.#startsInFlight.add(this.#inFlightKey(workflowId, runId)); try { await this.#reserveRunOwner(runId, source.owner, recovery.token); let summary: RunSummary; @@ -1389,6 +1577,7 @@ export abstract class DurableObjectRunner { requestedBy: principal.id, requestedByKind: principal.kind, attemptToken: recovery.token, + ...(idempotencyKey === undefined ? {} : { idempotencyKey }), ...(body.deadlineMs === undefined ? {} : { deadlineMs: body.deadlineMs as number }), @@ -1442,7 +1631,7 @@ export abstract class DurableObjectRunner { summary, ); await this.#settleRunOwnerBestEffort(recovery, false, !reconciled); - // DL-018: the authoritative RunSummary is the run-progress frame; push it + // The authoritative RunSummary is the run-progress frame; push it // to any subscribed run-channel socket at this lifecycle boundary. this.#broadcastRunSummary(summary); return json(summary); @@ -1454,9 +1643,29 @@ export abstract class DurableObjectRunner { await this.#rearmRunOwnerRecovery(); } throw error; + } finally { + // Whatever happened, this object is no longer starting the run. The + // delete must be unconditional: an entry left behind would answer + // every later probe "live" for the lifetime of the isolate, turning a + // crashed start's honest UNRESOLVABLE into an endless PENDING. + this.#startsInFlight.delete(this.#inFlightKey(workflowId, runId)); } }); } + // The liveness probe, answered OUTSIDE #withOperationLock on purpose: the + // start it is asking about holds that lock for the whole first leg, so a + // probe that queued behind it would block for exactly as long as the run it + // was trying to describe — and time out reporting nothing. + if ( + request.method === 'GET' && + segments.length === 4 && + action === 'start-liveness' && + workflowId && + runId + ) { + this.#assertRunIdentity(workflowId, runId); + return json({ live: this.#isStartLive(workflowId, runId) }); + } if ( request.method === 'GET' && segments.length === 3 && @@ -1509,7 +1718,7 @@ export abstract class DurableObjectRunner { ); } // The trusted Worker already verified the run ticket and routed by - // ticket.runId to idFromName; re-bind to this instance's identity (INV-1) + // ticket.runId to idFromName; re-bind to this instance's identity // before accepting so a mis-routed upgrade is refused. this.#assertRunIdentity(workflowId, runId); const runtime = this.#ensureRuntime(); @@ -1518,7 +1727,7 @@ export abstract class DurableObjectRunner { const { 0: client, 1: server } = newWebSocketPair(); state.acceptWebSocket(server); // On-connect snapshot: seed the new subscriber with the current - // authoritative summary (DL-018) so it need not wait for the next + // authoritative summary so it need not wait for the next // lifecycle transition. Nothing to send if the run is not yet queryable. safeSend(server, runFrame(snapshot)); return new Response(null, { @@ -1540,6 +1749,14 @@ export abstract class DurableObjectRunner { const resumeData = this.#resumeData(body.resumeData); return this.#withOperationLock(async () => { const runtime = this.#ensureRuntime(); + // Refused here as well as inside resume(), so a fenced deployment + // answers before it takes the per-run lock. A drain still admits + // resumes — the suspended runs it is draining are waiting for exactly + // these — and proof-only admits its one nominated run. + const resumeFence = await this.#readExecutionFence(); + if (!admitsExistingRun(resumeFence, runId)) { + throw new ExecutionFencedError(resumeFence.state, 'run resume'); + } const summary = await runtime.resume(workflowId, runId, { step: body.step, resumeData, @@ -1554,7 +1771,7 @@ export abstract class DurableObjectRunner { runId, summary, ); - // DL-018: broadcast the post-resume authoritative summary. + // Broadcast the post-resume authoritative summary. this.#broadcastRunSummary(summary); return json(summary); }); diff --git a/packages/flowsafe/src/do-runner/execution-fence.test.ts b/packages/flowsafe/src/do-runner/execution-fence.test.ts new file mode 100644 index 0000000..9f4227e --- /dev/null +++ b/packages/flowsafe/src/do-runner/execution-fence.test.ts @@ -0,0 +1,805 @@ +// SPDX-License-Identifier: Apache-2.0 +import { InMemoryStore } from '@mastra/core/storage'; +import { describe, expect, it } from 'vitest'; +import { z } from 'zod'; + +import { + openSqlite, + type SqliteDatabase, + sqliteUnitDatabase, +} from '../../test-support/sqlite.js'; +// The raw table name and the state list come from the PROVISIONING PROTOCOL, +// which is their single home — `./execution-fence.js` deliberately does not +// re-export them (see its header), so a test that pinned them off the runtime +// module would be pinning a second copy. +import { + EXECUTION_FENCE_STATES, + EXECUTION_FENCE_TABLE, +} from '../deployment-identity-protocol.js'; +import { doErrorResponse } from './do-error-response.js'; +import { + admitsDrainableExecution, + admitsExistingRun, + admitsRunStart, + admitsWorkAuthoring, + type ExecutionFenceDatabase, + ExecutionFencedError, + type ExecutionFenceReading, + type ExecutionFenceState, + ExecutionFenceStore, + ExecutionFenceUnreadableError, + FenceTransitionConflictError, + InvalidExecutionFenceRequestError, +} from './execution-fence.js'; +import { init } from './init.js'; +import type { RunnerRuntime } from './runtime.js'; + +function fenceFixture(): { + sqlite: SqliteDatabase; + db: ExecutionFenceDatabase; + fence: ExecutionFenceStore; +} { + const sqlite = openSqlite(); + const db = sqliteUnitDatabase(sqlite) as ExecutionFenceDatabase; + return { sqlite, db, fence: new ExecutionFenceStore(db) }; +} + +/** The schema as SQLite records it — the evidence a read wrote no DDL. */ +function schemaSnapshot(sqlite: SqliteDatabase): unknown[] { + return sqlite + .prepare('SELECT type, name, sql FROM sqlite_master ORDER BY name') + .all(); +} + +function reading( + state: ExecutionFenceState, + extra: Omit = {}, +): ExecutionFenceReading { + return { state, ...extra }; +} + +describe('ExecutionFenceStore', () => { + it('reads a database with no fence table as open, and writes no DDL doing it', async () => { + // #given — a 0.19-era database: the fence table does not exist. + const { sqlite, fence } = fenceFixture(); + const before = schemaSnapshot(sqlite); + + // #when + const observed = await fence.read(); + + // #then — open, and NOTHING was created. A read path that emits + // `CREATE TABLE IF NOT EXISTS` is a write path wearing a read's name; it + // would make a fenced deployment mutate its own database to answer a + // question, and would turn a revoked-write incident into an outage. + expect(observed).toEqual({ state: 'open' }); + expect(schemaSnapshot(sqlite)).toEqual(before); + expect(before).toEqual([]); + }); + + it('reads a seeded-but-rowless table as open', async () => { + // #given — the table exists (a crash between DDL and the row). + const { db, fence } = fenceFixture(); + await fence.seed('open'); + await db.prepare(`DELETE FROM ${EXECUTION_FENCE_TABLE}`).run(); + + // #then + await expect(fence.read()).resolves.toEqual({ state: 'open' }); + }); + + it('seed() requires an explicit state and never overwrites an existing row', async () => { + // #given — a deployment seeded locked at birth. + const { fence } = fenceFixture(); + await fence.seed('migration-locked'); + + // #when — provisioning runs again (the already-owned early-return path). + await fence.seed('open'); + + // #then — the operator's state survives. An upsert here would silently + // reopen a fence a migration closed. + await expect(fence.read()).resolves.toEqual({ state: 'migration-locked' }); + + // #and — the state is a required argument with no default, so a migration + // host cannot forget it and silently get 'open'. + await expect( + (fence as unknown as { seed(state?: unknown): Promise }).seed(), + ).rejects.toBeInstanceOf(InvalidExecutionFenceRequestError); + }); + + it('transitions on a matching expected state', async () => { + // #given + const { fence } = fenceFixture(); + await fence.seed('open'); + + // #when + const next = await fence.transition({ expected: 'open', next: 'draining' }); + + // #then + expect(next).toEqual({ state: 'draining' }); + await expect(fence.read()).resolves.toEqual({ state: 'draining' }); + }); + + it('materializes the implicit-open row of a database that has no fence table', async () => { + // #given — a 0.19 database whose fence reads as open with no row at all. + const { fence } = fenceFixture(); + + // #when — the first transition is also the first write. + await fence.transition({ expected: 'open', next: 'draining' }); + + // #then + await expect(fence.read()).resolves.toEqual({ state: 'draining' }); + }); + + it('refuses a CAS whose expected state is stale, and reports the CURRENT one', async () => { + // #given — another control-plane actor already locked it. + const { fence } = fenceFixture(); + await fence.seed('open'); + await fence.transition({ expected: 'open', next: 'draining' }); + + // #when / #then — the conflict carries the state the loser must re-plan + // against, so it needs no second round trip to find out. + const refusal = await fence + .transition({ expected: 'open', next: 'migration-locked' }) + .catch((error: unknown) => error); + expect(refusal).toBeInstanceOf(FenceTransitionConflictError); + expect((refusal as FenceTransitionConflictError).status).toBe(409); + expect((refusal as FenceTransitionConflictError).reason).toEqual({ + code: 'FENCE_CAS_CONFLICT', + state: 'draining', + }); + await expect(fence.read()).resolves.toEqual({ state: 'draining' }); + }); + + it("requires a proofKey to enter 'proof-only', and rejects one anywhere else", async () => { + // #given + const { fence } = fenceFixture(); + await fence.seed('migration-locked'); + + // #then — no key, no proof state. + await expect( + fence.transition({ expected: 'migration-locked', next: 'proof-only' }), + ).rejects.toBeInstanceOf(InvalidExecutionFenceRequestError); + // #and — a key for a state that has no proof is a caller belief that is + // false, so it is refused rather than ignored. + await expect( + fence.transition({ + expected: 'migration-locked', + next: 'open', + proofKey: 'proof-1', + }), + ).rejects.toBeInstanceOf(InvalidExecutionFenceRequestError); + await expect(fence.read()).resolves.toEqual({ state: 'migration-locked' }); + }); + + it('clears the proof run on entry to and exit from proof-only', async () => { + // #given — a proof state already bound to a run. + const { fence } = fenceFixture(); + await fence.seed('migration-locked'); + await fence.transition({ + expected: 'migration-locked', + next: 'proof-only', + proofKey: 'proof-1', + }); + expect(await fence.recordProofRun('proof-1', 'run-1')).toBe(true); + await expect(fence.read()).resolves.toEqual({ + state: 'proof-only', + proofKey: 'proof-1', + proofRunId: 'run-1', + }); + + // #when — a SECOND proof attempt under a new key. + await fence.transition({ + expected: 'proof-only', + next: 'proof-only', + proofKey: 'proof-2', + }); + + // #then — the prior proof's run is gone; the new key admits a fresh one. + await expect(fence.read()).resolves.toEqual({ + state: 'proof-only', + proofKey: 'proof-2', + }); + + // #and — leaving proof-only clears both fields. + await fence.transition({ expected: 'proof-only', next: 'open' }); + await expect(fence.read()).resolves.toEqual({ state: 'open' }); + }); + + describe('recordProofRun', () => { + it('binds the first run, and admits the SAME run again on replay', async () => { + // #given + const { fence } = fenceFixture(); + await fence.seed('migration-locked'); + await fence.transition({ + expected: 'migration-locked', + next: 'proof-only', + proofKey: 'proof-1', + }); + + // #then — first write binds; a retry of the interrupted start converges + // on the same runId instead of deadlocking on its own earlier write. + expect(await fence.recordProofRun('proof-1', 'run-1')).toBe(true); + expect(await fence.recordProofRun('proof-1', 'run-1')).toBe(true); + }); + + it('refuses a DIFFERENT run once the proof is bound', async () => { + const { fence } = fenceFixture(); + await fence.seed('migration-locked'); + await fence.transition({ + expected: 'migration-locked', + next: 'proof-only', + proofKey: 'proof-1', + }); + expect(await fence.recordProofRun('proof-1', 'run-1')).toBe(true); + + expect(await fence.recordProofRun('proof-1', 'run-2')).toBe(false); + }); + + it('refuses when the fence moved between the admit-read and the write-back', async () => { + // #given — admitted under proof-only... + const { fence } = fenceFixture(); + await fence.seed('migration-locked'); + await fence.transition({ + expected: 'migration-locked', + next: 'proof-only', + proofKey: 'proof-1', + }); + + // #when — ...and the operator locked it again before the write-back. + await fence.transition({ + expected: 'proof-only', + next: 'migration-locked', + }); + + // #then — zero rows changed, so the caller refuses the start. + expect(await fence.recordProofRun('proof-1', 'run-1')).toBe(false); + }); + + it('refuses a key that is not the nominated one', async () => { + const { fence } = fenceFixture(); + await fence.seed('migration-locked'); + await fence.transition({ + expected: 'migration-locked', + next: 'proof-only', + proofKey: 'proof-1', + }); + + expect(await fence.recordProofRun('proof-other', 'run-1')).toBe(false); + }); + + it('answers "not admitted" on a database with no fence table', async () => { + const { fence } = fenceFixture(); + expect(await fence.recordProofRun('proof-1', 'run-1')).toBe(false); + }); + }); + + it('fails closed on a state name this build does not understand', async () => { + // #given — a table this build did not create: a hand-edited row, or one + // written by a NEWER flowsafe that added a state. The CHECK constraint is + // deliberately absent, which is exactly what such a database would look + // like from here. + const { sqlite, db } = fenceFixture(); + const fence = new ExecutionFenceStore(db); + sqlite.exec( + `CREATE TABLE ${EXECUTION_FENCE_TABLE} ( + id TEXT PRIMARY KEY, + state TEXT NOT NULL, + proof_key TEXT, + proof_run_id TEXT, + updated_at INTEGER NOT NULL + )`, + ); + sqlite.exec( + `INSERT INTO ${EXECUTION_FENCE_TABLE} (id, state, updated_at) + VALUES ('deployment', 'quiesced-v2-from-the-future', 0)`, + ); + + // #then — never 'open'. Answering "I do not understand this fence" with + // "there is no fence" is the one answer that must never be wrong. + await expect(fence.read()).rejects.toBeInstanceOf( + ExecutionFenceUnreadableError, + ); + }); + + it('reads a pre-0.20 database as open when the adapter wraps the SQLite error', async () => { + // #given — an adapter that reports its own message and carries the driver's + // text on `cause`, with the missing table at the ROOT of the chain. This is + // the shape that makes the difference load-bearing: matching only the TOP + // message would classify a correctly upgraded 0.19 database as unreadable, + // and every gated path on it would answer 503 permanently — the exact + // opposite of the upgrade rule. + const wrapped = new Error('D1_ERROR: query failed', { + cause: new Error( + `SqliteError: no such table: ${EXECUTION_FENCE_TABLE}`, + // Two links deep, because an adapter over a driver over SQLite is the + // normal number of wrappers, not the pathological one. + { cause: new Error(`no such table: ${EXECUTION_FENCE_TABLE}`) }, + ), + }); + const fence = new ExecutionFenceStore({ + prepare: () => ({ + bind: () => ({ + run: () => Promise.reject(wrapped), + all: () => Promise.reject(wrapped), + }), + run: () => Promise.reject(wrapped), + all: () => Promise.reject(wrapped), + }), + } as unknown as ExecutionFenceDatabase); + + // #then — open, and `recordProofRun` reaches the same conclusion: a + // database with no fence table cannot be in proof-only. + await expect(fence.read()).resolves.toEqual({ state: 'open' }); + await expect(fence.recordProofRun('proof-1', 'acme_r1')).resolves.toBe( + false, + ); + }); + + it('still degrades closed when a wrapped cause is a genuine fault', async () => { + // #given — the same wrapping shape, but the buried error is a real storage + // fault. Walking the chain must not turn every wrapped error into an open + // fence: only the missing TABLE reads as open. + const wrapped = new Error('D1_ERROR: query failed', { + cause: new Error('no such table: mastra_workflow_snapshot'), + }); + const fence = new ExecutionFenceStore({ + prepare: () => ({ + bind: () => ({ + run: () => Promise.reject(wrapped), + all: () => Promise.reject(wrapped), + }), + run: () => Promise.reject(wrapped), + all: () => Promise.reject(wrapped), + }), + } as unknown as ExecutionFenceDatabase); + + // #then + await expect(fence.read()).rejects.toBeInstanceOf( + ExecutionFenceUnreadableError, + ); + }); + + it('degrades closed when the missing table is mid-chain and the ROOT is a real fault', async () => { + // #given — a chain that MENTIONS the fence table on its way past, but whose + // innermost fault is something else: a failed migration that dropped the + // table and then hit a real storage error underneath it. Matching any link + // would read this as "there is no fence table, so the deployment is open" — + // which is the one conclusion that must never be reached from a fault. Only + // the root says what actually happened. + const wrapped = new Error('D1_ERROR: query failed', { + cause: new Error(`no such table: ${EXECUTION_FENCE_TABLE}`, { + cause: new Error('D1_ERROR: database is locked'), + }), + }); + const fence = new ExecutionFenceStore({ + prepare: () => ({ + bind: () => ({ + run: () => Promise.reject(wrapped), + all: () => Promise.reject(wrapped), + }), + run: () => Promise.reject(wrapped), + all: () => Promise.reject(wrapped), + }), + } as unknown as ExecutionFenceDatabase); + + // #then — unreadable, not open. `recordProofRun` agrees, for the same + // reason: nothing about a locked database says the fence is absent. + await expect(fence.read()).rejects.toBeInstanceOf( + ExecutionFenceUnreadableError, + ); + await expect( + fence.recordProofRun('proof-1', 'acme_r1'), + ).rejects.toBeInstanceOf(ExecutionFenceUnreadableError); + }); + + it('terminates on a cyclic cause chain rather than degrading into a hang', async () => { + // #given — an error whose cause is itself. The walk runs on the fence read + // that fronts every gated request, so it is bounded and cycle-aware. + const cyclic = new Error('D1_ERROR: query failed'); + (cyclic as { cause?: unknown }).cause = cyclic; + const fence = new ExecutionFenceStore({ + prepare: () => ({ + bind: () => ({ + run: () => Promise.reject(cyclic), + all: () => Promise.reject(cyclic), + }), + run: () => Promise.reject(cyclic), + all: () => Promise.reject(cyclic), + }), + } as unknown as ExecutionFenceDatabase); + + // #then — a decided answer, not a hang. + await expect(fence.read()).rejects.toBeInstanceOf( + ExecutionFenceUnreadableError, + ); + }); + + it('degrades closed when the read itself fails', async () => { + // #given — storage that answers every query with a fault (NOT the + // "no such table" that legitimately reads as open). + const fence = new ExecutionFenceStore({ + prepare: () => ({ + bind: () => ({ + bind: () => { + throw new Error('unreachable'); + }, + run: () => Promise.reject(new Error('D1_ERROR: network')), + all: () => Promise.reject(new Error('D1_ERROR: network')), + }), + run: () => Promise.reject(new Error('D1_ERROR: network')), + all: () => Promise.reject(new Error('D1_ERROR: network')), + }), + } as unknown as ExecutionFenceDatabase); + + // #then — a 503 that names the condition, never a silent open. + const error = await fence.read().catch((thrown: unknown) => thrown); + expect(error).toBeInstanceOf(ExecutionFenceUnreadableError); + expect((error as ExecutionFenceUnreadableError).status).toBe(503); + expect((error as ExecutionFenceUnreadableError).reason).toEqual({ + code: 'EXECUTION_FENCE_UNREADABLE', + }); + }); +}); + +describe('execution fence admission predicates', () => { + it('admits a run START only in open, or in proof-only with the exact key', () => { + expect(admitsRunStart(reading('open'))).toBe(true); + expect(admitsRunStart(reading('draining'))).toBe(false); + expect(admitsRunStart(reading('migration-locked'))).toBe(false); + + const proof = reading('proof-only', { proofKey: 'proof-1' }); + expect(admitsRunStart(proof, 'proof-1')).toBe(true); + expect(admitsRunStart(proof, 'proof-2')).toBe(false); + expect(admitsRunStart(proof)).toBe(false); + // A proof state with no key admits nothing — never "any key matches". + expect(admitsRunStart(reading('proof-only'), 'proof-1')).toBe(false); + }); + + it('admits work on an EXISTING run through a drain, and in proof-only only for the proof run', () => { + expect(admitsExistingRun(reading('open'), 'run-1')).toBe(true); + expect(admitsExistingRun(reading('draining'), 'run-1')).toBe(true); + expect(admitsExistingRun(reading('migration-locked'), 'run-1')).toBe(false); + + const proof = reading('proof-only', { + proofKey: 'proof-1', + proofRunId: 'run-1', + }); + expect(admitsExistingRun(proof, 'run-1')).toBe(true); + expect(admitsExistingRun(proof, 'run-2')).toBe(false); + expect(admitsExistingRun(proof)).toBe(false); + expect(admitsExistingRun(reading('proof-only'), 'run-1')).toBe(false); + }); + + it('admits AUTHORING future work only while open', () => { + expect(admitsWorkAuthoring(reading('open'))).toBe(true); + for (const state of [ + 'draining', + 'migration-locked', + 'proof-only', + ] as const) { + expect(admitsWorkAuthoring(reading(state))).toBe(false); + } + }); + + it('admits draining execution of already-queued work, but not past the lock', () => { + expect(admitsDrainableExecution(reading('open'))).toBe(true); + expect(admitsDrainableExecution(reading('draining'))).toBe(true); + expect(admitsDrainableExecution(reading('migration-locked'))).toBe(false); + expect(admitsDrainableExecution(reading('proof-only'))).toBe(false); + }); + + it('covers every declared state', () => { + // A new state must be adjudicated by every predicate above rather than + // falling through one of them by default. + expect([...EXECUTION_FENCE_STATES]).toEqual([ + 'open', + 'draining', + 'migration-locked', + 'proof-only', + ]); + }); +}); + +describe('doErrorResponse', () => { + it('maps an invalid fence request to 400 with its reason code', async () => { + const error = new InvalidExecutionFenceRequestError('state is invalid'); + const response = doErrorResponse(error); + + expect(error.status).toBe(400); + expect(error.reason.code).toBe('INVALID_EXECUTION_FENCE_REQUEST'); + expect(response.status).toBe(400); + expect(await response.json()).toEqual({ + error: 'state is invalid', + reason: { code: 'INVALID_EXECUTION_FENCE_REQUEST' }, + }); + }); + + it('maps an ExecutionFencedError to 503 with its reason code', async () => { + const response = doErrorResponse( + new ExecutionFencedError('migration-locked', 'run start'), + ); + + expect(response.status).toBe(503); + expect(await response.json()).toEqual({ + error: + "deployment execution is fenced ('migration-locked'): run start is refused", + reason: { code: 'EXECUTION_FENCED', state: 'migration-locked' }, + }); + }); + + it('maps an unreadable fence to 503 with its own code', async () => { + const response = doErrorResponse( + new ExecutionFenceUnreadableError( + 'execution fence state is not readable', + ), + ); + + expect(response.status).toBe(503); + expect(await response.json()).toEqual({ + error: 'execution fence state is not readable', + reason: { code: 'EXECUTION_FENCE_UNREADABLE' }, + }); + }); +}); + +describe('init() fence wiring', () => { + it('auto-builds a fence from a { DB } source', async () => { + // #given — the shape every production host passes. + const { sqlite, db } = fenceFixture(); + const { runtime, executionFence } = init({ + DB: db as never, + }); + + // #then — fenced by construction: the host asked for nothing. + expect(executionFence).toBeInstanceOf(ExecutionFenceStore); + expect(runtime.executionFence).toBe(executionFence); + + // #and — it is THE SAME database, so nothing can fence one and read another. + await executionFence?.seed('draining'); + await expect(runtime.executionFence?.read()).resolves.toEqual({ + state: 'draining', + }); + expect( + schemaSnapshot(sqlite).some( + (row) => (row as { name?: string }).name === EXECUTION_FENCE_TABLE, + ), + ).toBe(true); + }); + + it("accepts an explicit 'none' for a { storage } source", () => { + const { runtime } = init( + { storage: new InMemoryStore() }, + { startIdempotency: 'none', executionFence: 'none' }, + ); + + expect(runtime.executionFence).toBeUndefined(); + }); + + it('takes a shared store for a { storage } source', async () => { + const { fence } = fenceFixture(); + const { runtime } = init( + { storage: new InMemoryStore() }, + { startIdempotency: 'none', executionFence: fence }, + ); + + expect(runtime.executionFence).toBe(fence); + }); + + it('will not compile a { storage } source without explicit fence wiring', () => { + // A TYPE-level pin. An UNUSED @ts-expect-error is itself an error in this + // package's tsconfig, so `tsc` exiting 0 is what proves the negative: the + // options argument is required, and omitting `executionFence` from it + // fails. This is the compile-time obligation that keeps a host from + // silently building an unfenced runtime. + const build = (): unknown => + // @ts-expect-error a { storage } source must state its fence wiring + init({ storage: new InMemoryStore() }); + expect(build).toBeTypeOf('function'); + }); +}); + +// The workflow the enforcement matrix drives: one step that suspends until it +// is resumed, so a single fixture covers both start and resume. +function fencedRuntime(fence: ExecutionFenceStore): RunnerRuntime { + const { createWorkflow, createStep, runtime } = init( + { storage: new InMemoryStore() }, + { startIdempotency: 'none', executionFence: fence }, + ); + const gate = createStep({ + id: 'gate', + inputSchema: z.object({}), + outputSchema: z.object({ done: z.boolean() }), + suspendSchema: z.object({ reason: z.string() }), + resumeSchema: z.object({ ok: z.boolean() }), + execute: async ({ resumeData, suspend }) => + resumeData ? { done: true } : suspend({ reason: 'awaiting' }), + }); + createWorkflow({ + id: 'gated', + inputSchema: z.object({}), + outputSchema: z.object({ done: z.boolean() }), + }) + .then(gate) + .commit(); + return runtime; +} + +describe('RunnerRuntime enforcement', () => { + it('starts and resumes freely while open', async () => { + const { fence } = fenceFixture(); + await fence.seed('open'); + const runtime = fencedRuntime(fence); + + const started = await runtime.start('gated', { + runId: 'run-open', + inputData: {}, + }); + expect(started.status).toBe('suspended'); + const resumed = await runtime.resume('gated', 'run-open', { + resumeData: { ok: true }, + }); + expect(resumed.status).toBe('success'); + }); + + it('blocks a start but still resumes while draining', async () => { + // #given — a run already suspended before the drain began. + const { fence } = fenceFixture(); + await fence.seed('open'); + const runtime = fencedRuntime(fence); + await runtime.start('gated', { runId: 'run-drain', inputData: {} }); + await fence.transition({ expected: 'open', next: 'draining' }); + + // #then — no new work... + const refusal = await runtime + .start('gated', { runId: 'run-drain-2' }) + .catch((error: unknown) => error); + expect(refusal).toBeInstanceOf(ExecutionFencedError); + expect((refusal as ExecutionFencedError).reason).toEqual({ + code: 'EXECUTION_FENCED', + state: 'draining', + }); + + // #and — ...but the outstanding run still finishes, which is the entire + // point of a drain. + const resumed = await runtime.resume('gated', 'run-drain', { + resumeData: { ok: true }, + }); + expect(resumed.status).toBe('success'); + }); + + it('blocks both a start and a resume under migration-locked', async () => { + const { fence } = fenceFixture(); + await fence.seed('open'); + const runtime = fencedRuntime(fence); + await runtime.start('gated', { runId: 'run-locked', inputData: {} }); + await fence.transition({ expected: 'open', next: 'migration-locked' }); + + await expect( + runtime.start('gated', { runId: 'run-locked-2', inputData: {} }), + ).rejects.toBeInstanceOf(ExecutionFencedError); + await expect( + runtime.resume('gated', 'run-locked', { resumeData: { ok: true } }), + ).rejects.toBeInstanceOf(ExecutionFencedError); + }); + + it('admits exactly the proof start and the proof resume under proof-only', async () => { + // #given + const { fence } = fenceFixture(); + await fence.seed('migration-locked'); + await fence.transition({ + expected: 'migration-locked', + next: 'proof-only', + proofKey: 'proof-key-1', + }); + const runtime = fencedRuntime(fence); + + // #then — a start with no key, or the wrong key, is refused. + await expect( + runtime.start('gated', { runId: 'proof-run', inputData: {} }), + ).rejects.toBeInstanceOf(ExecutionFencedError); + await expect( + runtime.start('gated', { + runId: 'proof-run', + idempotencyKey: 'guessed-key', + inputData: {}, + }), + ).rejects.toBeInstanceOf(ExecutionFencedError); + + // #and — the nominated start is admitted, and BINDS the proof run. + const started = await runtime.start('gated', { + runId: 'proof-run', + idempotencyKey: 'proof-key-1', + inputData: {}, + }); + expect(started.status).toBe('suspended'); + await expect(fence.read()).resolves.toEqual({ + state: 'proof-only', + proofKey: 'proof-key-1', + proofRunId: 'proof-run', + }); + + // #and — a SECOND start under the same key is refused: the proof is one + // run, and recordProofRun's CAS is what says so. + await expect( + runtime.start('gated', { + runId: 'other-run', + idempotencyKey: 'proof-key-1', + inputData: {}, + }), + ).rejects.toBeInstanceOf(ExecutionFencedError); + + // #and — only the proof run may be resumed. + await expect( + runtime.resume('gated', 'unrelated-run', { resumeData: { ok: true } }), + ).rejects.toBeInstanceOf(ExecutionFencedError); + const resumed = await runtime.resume('gated', 'proof-run', { + resumeData: { ok: true }, + }); + expect(resumed.status).toBe('success'); + }); + + it('lets a start already past its fence read complete when the fence moves', async () => { + // #given — a workflow whose FIRST step closes the fence mid-run, which is + // the in-flight race a transition must not preempt. + const { fence } = fenceFixture(); + await fence.seed('open'); + const { createWorkflow, createStep, runtime } = init( + { storage: new InMemoryStore() }, + { startIdempotency: 'none', executionFence: fence }, + ); + const drainMidRun = createStep({ + id: 'drain-mid-run', + inputSchema: z.object({}), + outputSchema: z.object({ done: z.boolean() }), + execute: async () => { + await fence.transition({ expected: 'open', next: 'draining' }); + return { done: true }; + }, + }); + createWorkflow({ + id: 'racing', + inputSchema: z.object({}), + outputSchema: z.object({ done: z.boolean() }), + }) + .then(drainMidRun) + .commit(); + + // #when + const summary = await runtime.start('racing', { + runId: 'run-racing', + inputData: {}, + }); + + // #then — in-flight compute is never preempted; only the NEXT start is + // refused. The drain sequence is drain, then prove empty, then lock. + expect(summary.status).toBe('success'); + await expect(fence.read()).resolves.toEqual({ state: 'draining' }); + await expect( + runtime.start('racing', { runId: 'run-racing-2', inputData: {} }), + ).rejects.toBeInstanceOf(ExecutionFencedError); + }); + + it('degrades a start closed when the fence cannot be read', async () => { + // #given — a fence whose storage is down. + const failing = new ExecutionFenceStore({ + prepare: () => ({ + bind: () => ({ + bind: () => { + throw new Error('unreachable'); + }, + run: () => Promise.reject(new Error('D1_ERROR: network')), + all: () => Promise.reject(new Error('D1_ERROR: network')), + }), + run: () => Promise.reject(new Error('D1_ERROR: network')), + all: () => Promise.reject(new Error('D1_ERROR: network')), + }), + } as unknown as ExecutionFenceDatabase); + const runtime = fencedRuntime(failing); + + // #then — 503, not a start on a deployment whose state is unknown. + const error = await runtime + .start('gated', { runId: 'run-unreadable' }) + .catch((thrown: unknown) => thrown); + expect(error).toBeInstanceOf(ExecutionFenceUnreadableError); + expect(doErrorResponse(error).status).toBe(503); + }); +}); diff --git a/packages/flowsafe/src/do-runner/execution-fence.ts b/packages/flowsafe/src/do-runner/execution-fence.ts new file mode 100644 index 0000000..d69d36f --- /dev/null +++ b/packages/flowsafe/src/do-runner/execution-fence.ts @@ -0,0 +1,783 @@ +// SPDX-License-Identifier: Apache-2.0 +// The deployment execution fence — the control that lets an operator stop a +// deployment from MINTING work without stopping it from finishing the work it +// already has. +// +// WHY it exists: a flowsafe deployment is one tenant's whole execution surface +// (one Worker, one D1, its own Durable Object namespaces). Moving that surface +// to another deployment is only safe if the old one can be brought to a state +// where nothing new starts, everything outstanding drains, and then nothing +// runs at all — otherwise the two deployments execute the same run's steps +// against the same rows and every exactly-once property this package defends +// dies at the migration boundary. +// +// The states are ordered by how much they forbid, and the ORDER is the whole +// contract: +// +// open everything (the steady state) +// draining finish what exists; mint nothing new +// migration-locked execute nothing; reads and admin still answer +// proof-only migration-locked, except ONE nominated run — the proof +// that the deployment still works before it is reopened +// +// Two invariants hold everywhere the fence is consulted: +// +// NEVER MEMOIZED. The fence is an operational control an operator moves +// between requests; a memo would serve a stale answer for the length of an +// isolate, which is exactly the window a migration is trying to close. One +// read per request, or one per tick/dispatch PASS — never per row. +// +// DEGRADE CLOSED. A read that did not reach storage is not evidence that the +// deployment is open. On a request path it is a 503 (the operator's problem, +// retryable — the same answer DeploymentIdentityError gets); on an alarm path +// it is logged, swallowed, and left for the next wake, because a thrown alarm +// is retried by workerd and would answer a storage incident with a storm. +// +// ABSENT ROW (and absent TABLE) READ AS `open`. That is the 0.19-to-0.20 +// upgrade rule and nothing more: a database seeded before this table existed +// must keep serving. Provisioning writes an explicit row from 0.20 on, so a +// deployment that means to start locked says so rather than relying on a +// default — `seed()` therefore takes the state as a REQUIRED argument. + +import { + EXECUTION_FENCE_DDL, + EXECUTION_FENCE_ROW_ID, + EXECUTION_FENCE_STATES, + EXECUTION_FENCE_TABLE, +} from '#deployment-identity-protocol'; +import { missingTableReadsEmpty } from './cause-chain.js'; +import { DoStatusError } from './do-status-error.js'; +import { isPathSafeId } from './path-safe-id.js'; + +/** + * Rows affected by a write, read from D1's `{ meta: { changes } }` envelope — + * the same accessor d1-storage exports as `d1Changes`, restated here so this + * module imports only leaf modules. Every surface that consults the fence + * imports it, including ones that must not drag the D1 storage adapter (and + * @mastra/cloudflare-d1 with it) into their bundle. + */ +function changesOf(result: unknown): number { + const changes = (result as { meta?: { changes?: number } } | undefined)?.meta + ?.changes; + return typeof changes === 'number' ? changes : 0; +} + +/** + * The state vocabulary, the table, that table's fixed row key, and the DDL + * built from all three are IMPORTED, never declared here — and they are not + * re-exported from this module either. + * + * Imported because PROVISIONING creates this table + * (deployment-identity-protocol.mjs), and `CREATE TABLE IF NOT EXISTS` would + * silently accept a differently-shaped table it had already made — so a second + * copy of the schema would not fail loudly, it would quietly drop the CHECK + * constraints that make every CAS below total rather than "the CAS, on + * whichever row you meant". That module is also the only file both sides can + * share: it ships at the package root for the provisioning CLI and + * fleet-control, neither of which can import this package's TypeScript. + * + * Not re-exported because a raw constant with two homes is a constant two + * consumers can disagree about. `@proofoftech/flowsafe/deployment-identity-protocol` + * is the one place to import the table name or the state list from; what this + * module publishes is the TYPED surface built on them — the state type, the + * store, the refusals, and the admission predicates. + */ +export type ExecutionFenceState = (typeof EXECUTION_FENCE_STATES)[number]; + +/** + * The suspend-payload key the executor backstop stamps on a task it parked + * because the deployment was fenced mid-dispatch. Namespaced so it cannot + * collide with a tool's own suspend payload, and read back by + * `BackgroundTaskHost.#resumeFenceSuspendedTasks` — which is what makes the + * parking reversible rather than a quieter kind of loss. + * + * PUBLISHED via `./background-tasks` because it is the only way a host can tell + * a fence-parked row from a tool-suspended one: `listTasks({ status: + * 'suspended' })` returns both, and the marker lives in the suspend payload + * where no filter can express it. The drain inventory imports this same + * definition directly, so a rename cannot silently leave a hard-coded census + * predicate behind. + */ +export const EXECUTION_FENCE_SUSPEND_KEY = 'flowsafe.executionFenced'; + +/** + * How a surface is wired to the fence: a store, or the typed opt-out. + * + * Written as a union with no `undefined` so every option type carrying it can + * be REQUIRED. That is the whole forcing function: a fence option a host may + * omit is one a host will omit, and a partially wired deployment is worse than + * an unwired one — an unfenced schedule tick claims a due fire through the CAS + * (which advances `nextFireAt`) and the fenced runtime then refuses the start, + * so the fire is consumed and never runs. Making the caller WRITE `'none'` + * turns that split brain into a decision someone made rather than one they + * missed, and `'none'` stays honest for the callers that genuinely have no + * database to fence against (in-memory tests, adapters). + */ +export type ExecutionFenceWiring = ExecutionFenceStore | 'none'; + +/** What one fence read observed. `proofKey`/`proofRunId` exist only in proof-only. */ +export interface ExecutionFenceReading { + readonly state: ExecutionFenceState; + /** The key a proof-only start must carry to be admitted. */ + readonly proofKey?: string; + /** The run the proof-only state has already admitted, once one started. */ + readonly proofRunId?: string; +} + +/** The reading every unfenced surface uses — see `ExecutionFenceStore` absence. */ +export const OPEN_EXECUTION_FENCE: ExecutionFenceReading = Object.freeze({ + state: 'open', +}); + +/** + * Read the fence a surface was wired with, resolving the typed opt-out. + * + * ONE function rather than a `fence ? await fence.read() : OPEN` at every gate, + * because those are the places a mistake is invisible: an unfenced surface and + * an open one behave identically until the day an operator closes the fence, so + * a call site that got the ternary subtly wrong would pass every test written + * against an open deployment. Every gate resolving absence through here means + * there is exactly one definition of what "no fence" does. + * + * `undefined` is admitted alongside `'none'` for the surfaces whose fence + * arrives through an object the caller may not have populated (an + * `InitResult.executionFence` on an unfenced host); it reads as open for the + * same reason `'none'` does, and never as a silent default a host can reach by + * forgetting — the option types that feed this are required. + */ +export async function readExecutionFence( + fence: ExecutionFenceWiring | undefined, +): Promise { + if (fence === undefined || fence === 'none') return OPEN_EXECUTION_FENCE; + return fence.read(); +} + +/** + * One fence store per DATABASE — not per request, and not per call site. + * + * The store holds no state of its own (a read is never memoized — see the + * module header), so this is about IDENTITY rather than caching: the admin + * route that MOVES the fence, the approval service that OBEYS it, the schedule + * tick that must not claim a fire behind it, and the runner's runtime all have + * to be looking at the same database. Keyed on the BINDING rather than on an + * env object, because a host mutates one env across requests and because the + * fence belongs to the database, not to the request that happens to reach it. + * WeakMap, so a test harness cycling bindings never leaks. + * + * Lives here rather than once per host because every composer needed the same + * three lines, and four copies of a memo are four chances for one of them to + * key on the wrong thing — an env-keyed copy hands two databases the same + * fence. A host whose call sites pass `env` keeps a one-line wrapper of its + * own; the memo itself is this one. + */ +const executionFenceStores = new WeakMap(); + +export function executionFenceFor( + db: ExecutionFenceDatabase, +): ExecutionFenceStore { + const existing = executionFenceStores.get(db); + if (existing) return existing; + const store = new ExecutionFenceStore(db); + executionFenceStores.set(db, store); + return store; +} + +/** + * Minimal structural D1 surface, the same posture as SnapshotDatabase and + * ApprovalDatabase: tests back it with node:sqlite, Workers pass env.DB. + * + * `all()` rather than `first()` for the one read, deliberately: this exact + * shape is what DeploymentIdentityDatabase, SnapshotDatabase, and + * ApprovalDatabase all already satisfy, so every surface that must consult the + * fence can hand over the binding it already holds with no cast and no second + * seam. The fence's row is unique by primary key, so the two are equivalent. + */ +export interface ExecutionFenceDatabase { + prepare(query: string): ExecutionFenceStatement; +} + +export interface ExecutionFenceStatement { + bind(...values: unknown[]): ExecutionFenceStatement; + run(): Promise; + all(): Promise<{ results: T[] }>; +} + +/** + * The refusal a fenced surface answers with: 503, because the deployment is + * deliberately not executing right now and will be again — an operator's + * condition, not the caller's mistake and not a code fault. Providers, + * schedulers, and clients that honour Retry-After semantics therefore + * redeliver rather than discard. + */ +export class ExecutionFencedError extends DoStatusError { + readonly status = 503; + readonly reason: { + readonly code: 'EXECUTION_FENCED'; + readonly state: ExecutionFenceState; + }; + + constructor(state: ExecutionFenceState, entry?: string) { + super( + entry === undefined + ? `deployment execution is fenced ('${state}')` + : `deployment execution is fenced ('${state}'): ${entry} is refused`, + ); + this.name = 'ExecutionFencedError'; + this.reason = { code: 'EXECUTION_FENCED', state }; + } +} + +/** + * A fence transition whose compare-and-set found a different state. 409 rather + * than 503: the deployment is fine, the CALLER's expectation is stale — two + * control-plane actors raced, or an operator retried a transition that already + * landed. The current state rides on the reason so the caller can re-plan + * without a second round trip. + */ +export class FenceTransitionConflictError extends DoStatusError { + readonly status = 409; + readonly reason: { + readonly code: 'FENCE_CAS_CONFLICT'; + readonly state: ExecutionFenceState; + }; + + constructor(expected: ExecutionFenceState, current: ExecutionFenceState) { + super( + `execution fence transition expected state '${expected}' but found '${current}'`, + ); + this.name = 'FenceTransitionConflictError'; + this.reason = { code: 'FENCE_CAS_CONFLICT', state: current }; + } +} + +/** + * The fence could not be READ. Deliberately distinct from + * ExecutionFencedError: no state was observed, so nothing may conclude the + * deployment is open — which is why this carries the same 503 a refusal does + * and every request-path caller lets it propagate. + */ +export class ExecutionFenceUnreadableError extends DoStatusError { + readonly status = 503; + readonly reason: { readonly code: 'EXECUTION_FENCE_UNREADABLE' }; + + constructor(message: string, options?: ErrorOptions) { + super(message, options); + this.name = 'ExecutionFenceUnreadableError'; + this.reason = { code: 'EXECUTION_FENCE_UNREADABLE' }; + } +} + +/** + * Render a fence refusal as the JSON response the taxonomy specifies. + * + * For the surfaces that answer with a Response instead of throwing — a Worker + * router, or a Durable Object route whose own catch would re-map a thrown + * status. Built from the ERROR so the body a router writes and the body + * doErrorResponse writes for the same refusal cannot drift apart. + */ +export function executionFencedResponse( + state: ExecutionFenceState, + entry?: string, +): Response { + const refusal = new ExecutionFencedError(state, entry); + return new Response( + JSON.stringify({ error: refusal.message, reason: refusal.reason }), + { + status: refusal.status, + headers: { + 'content-type': 'application/json', + 'cache-control': 'no-store', + }, + }, + ); +} + +/** + * A fence reading as the JSON body a control-plane read answers with. + * + * Beside `executionFencedResponse` and for the same reason: the published + * `GET /admin/execution-fence` route and the spike's local control probe both + * project a reading into this exact shape, and a projection written twice is + * one an operator's tooling can watch drift. Absent fields are OMITTED rather + * than sent as null — `proofKey`/`proofRunId` exist only in proof-only, and a + * null would invite a caller to read "no proof run yet" out of a state that has + * no proof at all. + */ +export function executionFenceReadingPayload(reading: ExecutionFenceReading): { + state: ExecutionFenceState; + proofKey?: string; + proofRunId?: string; +} { + return { + state: reading.state, + ...(reading.proofKey === undefined ? {} : { proofKey: reading.proofKey }), + ...(reading.proofRunId === undefined + ? {} + : { proofRunId: reading.proofRunId }), + }; +} + +/** + * A fence refusal that CROSSED a Durable Object boundary and was rebuilt on the + * far side. + * + * The run object throws ExecutionFencedError, `doErrorResponse` renders it, and + * `doSummary` reconstructs it as a `RunRouteError` carrying the same status, + * message and structured reason — but not the same class. This structural arm + * is what lets a Worker-side caller treat that reconstruction as the refusal it + * is, with the same three fields every render site already reads. + */ +export interface WireExecutionFenceRefusal { + readonly message: string; + readonly status: number; + readonly reason: { readonly code: string }; +} + +/** Every fence-authored refusal — the family a fenced surface catches as one. */ +export type ExecutionFenceRefusal = + | ExecutionFencedError + | ExecutionFenceUnreadableError + | WireExecutionFenceRefusal; + +/** The two reason codes a fence refusal publishes, whatever carried it. */ +const FENCE_REFUSAL_CODES: ReadonlySet = new Set([ + 'EXECUTION_FENCED', + 'EXECUTION_FENCE_UNREADABLE', +]); + +/** + * Whether an error is the fence refusing (or failing to answer). The two are + * one class for every CALLER that must degrade closed — an alarm swallowing + * both, a tick skipping its pass — because a state it could not read and a + * state that forbids the work lead to the same action. + * + * The CODE counts as well as the class, because a refusal that crossed a + * Durable Object boundary is no longer an instance of anything: the run object + * threw ExecutionFencedError, `doErrorResponse` rendered it, and `doSummary` + * rebuilt it on the Worker side as a `RunRouteError` carrying the same status + * and the same structured reason. That rebuilt error IS the fence refusing — + * the callers most in need of recognizing one (a run router deciding whether to + * give a reservation's claim back, a tick deciding whether to skip a pass) are + * exactly the ones sitting on the far side of that boundary, and an + * instanceof-only test would answer "no" for every one of them. + * + * The two codes are matched by name rather than by any structural sniff: only + * refusals this package authors publish them, and both are declared as literals + * on the classes above, so a code arriving over the wire came from one of them. + */ +export function isExecutionFenceRefusal( + error: unknown, +): error is ExecutionFenceRefusal { + if ( + error instanceof ExecutionFencedError || + error instanceof ExecutionFenceUnreadableError + ) { + return true; + } + if (error === null || typeof error !== 'object') return false; + const { message, status, reason } = error as { + message?: unknown; + status?: unknown; + reason?: unknown; + }; + // All three fields are checked, not just the code: this predicate NARROWS, + // and every render site immediately reads message/status/reason off what it + // narrowed. Asserting a shape on the strength of one field would hand them a + // `status` of undefined, and `new Response(body, { status: undefined })` + // fails inside the very catch block whose job is to never throw. + if (typeof message !== 'string' || !Number.isInteger(status)) return false; + if (reason === null || typeof reason !== 'object') return false; + const { code } = reason as { code?: unknown }; + return typeof code === 'string' && FENCE_REFUSAL_CODES.has(code); +} + +function isExecutionFenceState(value: unknown): value is ExecutionFenceState { + return ( + typeof value === 'string' && + (EXECUTION_FENCE_STATES as readonly string[]).includes(value) + ); +} + +/** + * Validate a state name from a control-plane request. Exported because the + * admin route validates the WIRE shape and the package validates nothing else + * about a transition: which transitions are legal is host policy, and this + * package only enforces the state vocabulary, the CAS, and the proof-only + * key requirement. + */ +export function assertExecutionFenceState( + value: unknown, + field: string, +): ExecutionFenceState { + if (!isExecutionFenceState(value)) { + throw new InvalidExecutionFenceRequestError( + `${field} must be one of ${EXECUTION_FENCE_STATES.join(', ')}`, + ); + } + return value; +} + +/** A malformed control-plane fence request — the caller's to fix. */ +export class InvalidExecutionFenceRequestError extends DoStatusError { + readonly status = 400; + readonly reason: { readonly code: 'INVALID_EXECUTION_FENCE_REQUEST' }; + + constructor(message: string) { + super(message); + this.name = 'InvalidExecutionFenceRequestError'; + this.reason = { code: 'INVALID_EXECUTION_FENCE_REQUEST' }; + } +} + +// --------------------------------------------------------------------------- +// Admission predicates — the semantics matrix, as four total functions. +// +// One predicate per COLUMN of behaviour rather than one per call site, so two +// surfaces that must answer the same way cannot drift apart. Resume, approval +// decide, and signal delivery share `admitsExistingRun` for exactly that +// reason: all three act on a run that already exists, so all three must stay +// open through a drain and all three must admit the proof run and nothing else. +// --------------------------------------------------------------------------- + +/** + * May a NEW run be minted? Only `open`, plus the proof-only exception: a start + * whose idempotency key is the nominated proof key. The key is internal + * material (never a request-body field, never an open request-context key), so + * nothing a tenant can send reaches this branch. + */ +export function admitsRunStart( + reading: ExecutionFenceReading, + idempotencyKey?: string, +): boolean { + if (reading.state === 'open') return true; + if (reading.state !== 'proof-only') return false; + return ( + reading.proofKey !== undefined && + idempotencyKey !== undefined && + idempotencyKey === reading.proofKey + ); +} + +/** + * May work proceed on a run that ALREADY exists — resume, approval decide, + * signal delivery? Through a drain, yes: a drain that refused these could + * never finish, because finishing is what the suspended runs are waiting for. + * In proof-only, only the nominated run. + */ +export function admitsExistingRun( + reading: ExecutionFenceReading, + runId?: string, +): boolean { + if (reading.state === 'open' || reading.state === 'draining') return true; + if (reading.state !== 'proof-only') return false; + return ( + reading.proofRunId !== undefined && + runId !== undefined && + runId === reading.proofRunId + ); +} + +/** + * May standing configuration that ARMS future work be authored — a schedule + * created/updated/resumed, an objective set, a due schedule fire claimed? + * `open` only. Pausing and deleting stay allowed in every state: they remove + * future work, which is the direction a drain is going. + */ +export function admitsWorkAuthoring(reading: ExecutionFenceReading): boolean { + return reading.state === 'open'; +} + +/** + * May queued, already-owned work be executed — a background task body, a + * notification dispatch pass? Through a drain, yes; that queue IS the work a + * drain exists to finish. Not in proof-only: the proof is one run, and a task + * queue re-driving itself alongside it is not a proof of anything. + */ +export function admitsDrainableExecution( + reading: ExecutionFenceReading, +): boolean { + return reading.state === 'open' || reading.state === 'draining'; +} + +interface ExecutionFenceRow { + state?: unknown; + proof_key?: unknown; + proof_run_id?: unknown; +} + +function readingFromRow(row: ExecutionFenceRow): ExecutionFenceReading { + const { state } = row; + if (!isExecutionFenceState(state)) { + // Fail CLOSED on a state name this build does not know: a hand-edited row, + // or a row written by a NEWER flowsafe that added a state. Returning + // `open` for either would answer "I do not understand this fence" with + // "there is no fence", which is the one answer that must never be wrong. + throw new ExecutionFenceUnreadableError( + `execution fence row carries an unrecognized state '${String(state)}'`, + ); + } + const proofKey = row.proof_key; + const proofRunId = row.proof_run_id; + return { + state, + ...(typeof proofKey === 'string' && proofKey.length > 0 + ? { proofKey } + : {}), + ...(typeof proofRunId === 'string' && proofRunId.length > 0 + ? { proofRunId } + : {}), + }; +} + +/** + * SQLite/D1's "no such table", for THIS store's table: a table that was never + * created is not a fault here — it is a pre-0.20 database, which reads as + * `open`. + * + * The rule itself (walk the cause chain, bounded and cycle-safe, and test the + * ROOT only) lives in cause-chain.ts, shared with the start-reservation store + * because both answer the same question and both are dangerous in the same + * direction. A chain with no reachable root degrades CLOSED there for the same + * reason it does here: no root was observed, and an unobserved root is not + * evidence of an absent table. + */ +function isMissingFenceTable(error: unknown): boolean { + return missingTableReadsEmpty(error, EXECUTION_FENCE_TABLE); +} + +export interface ExecutionFenceStoreOptions { + /** Injectable clock for `updated_at` (tests, deterministic fixtures). */ + now?: () => number; +} + +export interface ExecutionFenceTransition { + /** The state the caller believes the fence is in. */ + expected: ExecutionFenceState; + /** The state to move to. */ + next: ExecutionFenceState; + /** + * Required when `next` is 'proof-only'; rejected otherwise. + * + * `unknown` rather than `string` because every caller is a control-plane + * route holding a parsed JSON body, and `#proofKeyFor` already validates it + * against PATH_SAFE_ID_PATTERN and throws InvalidExecutionFenceRequestError + * on anything else. Typing it `string` bought nothing and cost something: it + * made every route write `body.proofKey as string`, an assertion that is + * false exactly when the caller sent the wrong thing, so the one input this + * field exists to police arrived pre-blessed at the type level. + */ + proofKey?: unknown; +} + +/** + * The deployment's fence, over one D1 database — the SAME database the runner's + * snapshots and the deployment sentinel live in, so the fence cannot be + * separated from the state it fences by any binding mistake. + */ +export class ExecutionFenceStore { + readonly #db: ExecutionFenceDatabase; + readonly #now: () => number; + + constructor( + db: ExecutionFenceDatabase, + options: ExecutionFenceStoreOptions = {}, + ) { + this.#db = db; + this.#now = options.now ?? Date.now; + } + + /** + * The current fence state. NEVER memoized (see the module header) and never + * a write: this is on every gated request path, and a read path that emits + * `CREATE TABLE IF NOT EXISTS` is a write path wearing a read's name — it + * would make a fenced deployment mutate its own database to answer a + * question, and would turn a read-only replica or a revoked-write incident + * into an outage instead of a degrade. + * + * A missing table and a missing row both read as `open` — the 0.19 upgrade + * rule. Anything else that fails becomes ExecutionFenceUnreadableError, so + * no caller can mistake a storage fault for an open deployment. + */ + async read(): Promise { + let rows: ExecutionFenceRow[]; + try { + rows = ( + await this.#db + .prepare( + `SELECT state, proof_key, proof_run_id FROM ${EXECUTION_FENCE_TABLE} + WHERE id = ?`, + ) + .bind(EXECUTION_FENCE_ROW_ID) + .all() + ).results; + } catch (error) { + if (isMissingFenceTable(error)) return OPEN_EXECUTION_FENCE; + throw new ExecutionFenceUnreadableError( + 'execution fence state is not readable', + { cause: error }, + ); + } + const row = rows[0]; + if (row === undefined) return OPEN_EXECUTION_FENCE; + return readingFromRow(row); + } + + /** + * Provisioning-time seeding: write the deployment's INITIAL fence state. + * + * INSERT-if-absent, never an overwrite. Seeding runs on every provisioning + * pass (including the already-owned early return), so a crash between the + * deployment sentinel and this row heals on the next attempt — but a + * re-provision of a LIVE deployment must never silently reopen a fence an + * operator closed, which an upsert would do. + * + * `state` has no default on purpose. The failure this closes is a migration + * host forgetting to ask for `migration-locked` and silently getting `open`; + * making the argument required turns that into a compile-time obligation, + * while still letting a host that wants an open deployment say so. + */ + async seed(state: ExecutionFenceState): Promise { + const safeState = assertExecutionFenceState(state, 'seed state'); + await this.#createTable(); + await this.#db + .prepare( + `INSERT OR IGNORE INTO ${EXECUTION_FENCE_TABLE} + (id, state, proof_key, proof_run_id, updated_at) + VALUES (?, ?, NULL, NULL, ?)`, + ) + .bind(EXECUTION_FENCE_ROW_ID, safeState, this.#now()) + .run(); + } + + /** + * Move the fence, compare-and-set on the CURRENT state. One conditional + * UPDATE, so two control-plane actors racing the same transition cannot both + * win: the loser changes zero rows and gets the state the winner left behind. + * + * Unlike `read()` this MAY create the table — a transition is a control-plane + * write, and a legacy database whose fence is implicitly open has no row to + * compare against. The row is materialized as `open` first, which is the + * state the implicit reading already reported, so the CAS that follows means + * exactly what it would have meant on a seeded database. + */ + async transition( + input: ExecutionFenceTransition, + ): Promise { + const expected = assertExecutionFenceState( + input.expected, + 'expected state', + ); + const next = assertExecutionFenceState(input.next, 'next state'); + const proofKey = this.#proofKeyFor(next, input.proofKey); + await this.#createTable(); + // Materialize the implicit-open row of a pre-0.20 database. INSERT OR + // IGNORE, so a seeded database is untouched and the CAS below is still the + // only thing that decides the outcome. + await this.#db + .prepare( + `INSERT OR IGNORE INTO ${EXECUTION_FENCE_TABLE} + (id, state, proof_key, proof_run_id, updated_at) + VALUES (?, 'open', NULL, NULL, ?)`, + ) + .bind(EXECUTION_FENCE_ROW_ID, this.#now()) + .run(); + // proof_run_id is cleared unconditionally: ENTERING proof-only must not + // inherit a previous proof's run, and LEAVING it must not leave a stale + // admission behind for the next one to trip over. + const changed = changesOf( + await this.#db + .prepare( + `UPDATE ${EXECUTION_FENCE_TABLE} + SET state = ?, proof_key = ?, proof_run_id = NULL, updated_at = ? + WHERE id = ? AND state = ?`, + ) + .bind( + next, + proofKey ?? null, + this.#now(), + EXECUTION_FENCE_ROW_ID, + expected, + ) + .run(), + ); + if (changed === 0) { + throw new FenceTransitionConflictError( + expected, + (await this.read()).state, + ); + } + return { state: next, ...(proofKey === undefined ? {} : { proofKey }) }; + } + + /** + * Bind the proof-only state to the run it admitted, conditionally. + * + * The condition is the whole point: between the read that ADMITTED a start + * and this write-back the fence may have moved, or a different run may have + * claimed the proof. Zero rows changed means the caller must refuse the + * start it was about to make — the fence is no longer the one it read. + * Re-writing the SAME runId is admitted so a retry of an interrupted start + * converges instead of deadlocking on its own earlier write. + */ + async recordProofRun(proofKey: string, runId: string): Promise { + if (!isPathSafeId(proofKey)) { + throw new InvalidExecutionFenceRequestError( + 'proofKey must be a URL-path-safe identifier', + ); + } + if (!isPathSafeId(runId)) { + throw new InvalidExecutionFenceRequestError( + 'proof runId must be a URL-path-safe identifier', + ); + } + try { + return ( + changesOf( + await this.#db + .prepare( + `UPDATE ${EXECUTION_FENCE_TABLE} + SET proof_run_id = ?, updated_at = ? + WHERE id = ? AND state = 'proof-only' AND proof_key = ? + AND (proof_run_id IS NULL OR proof_run_id = ?)`, + ) + .bind(runId, this.#now(), EXECUTION_FENCE_ROW_ID, proofKey, runId) + .run(), + ) > 0 + ); + } catch (error) { + // A database with no fence table cannot be in proof-only, so there is + // nothing to record and nothing to conclude beyond "not admitted". + if (isMissingFenceTable(error)) return false; + throw new ExecutionFenceUnreadableError( + 'execution fence proof run could not be recorded', + { cause: error }, + ); + } + } + + #proofKeyFor( + next: ExecutionFenceState, + proofKey: unknown, + ): string | undefined { + if (next === 'proof-only') { + if (!isPathSafeId(proofKey)) { + throw new InvalidExecutionFenceRequestError( + "a URL-path-safe proofKey is required to enter 'proof-only'", + ); + } + return proofKey; + } + if (proofKey !== undefined) { + // Rejected rather than ignored: a caller passing a key for a state that + // has no proof believes something about this transition that is false. + throw new InvalidExecutionFenceRequestError( + `proofKey applies only to 'proof-only' transitions, not '${next}'`, + ); + } + return undefined; + } + + async #createTable(): Promise { + await this.#db.prepare(EXECUTION_FENCE_DDL).run(); + } +} diff --git a/packages/flowsafe/src/do-runner/index.ts b/packages/flowsafe/src/do-runner/index.ts index 9346cd3..342ba90 100644 --- a/packages/flowsafe/src/do-runner/index.ts +++ b/packages/flowsafe/src/do-runner/index.ts @@ -39,6 +39,7 @@ export { purgeExpiredThreadState, purgeExpiredThreads, purgeExpiredWorkflowRuns, + RUN_TTL_FLOWSAFE_PURGE_TABLES, RUN_TTL_PURGE_TABLES, SCHEDULE_TRIGGER_TTL_PURGE_TABLES, sweepExpiredRunDeadlines, @@ -51,6 +52,7 @@ export type { DeploymentIdentityDatabase, DeploymentIdentityEnv, DeploymentIdentityStatement, + InitialExecutionFenceState, } from './deployment-identity.js'; export { assertDeploymentIdentity, @@ -69,6 +71,7 @@ export { } from './deployment-identity.js'; // The DO error taxonomy and its extension point: a host DO's own route states a // status by extending DoStatusError (see do-error-response.ts). +export type { DoRefusalReason } from './do-error-response.js'; export { DoStatusError, doErrorResponse } from './do-error-response.js'; export { type DurableObjectRunLifecycleHooks, @@ -84,6 +87,55 @@ export { ReservedExecutionContextError, stripReservedExecutionContext, } from './execution-context.js'; +// The deployment execution fence (docs/do-runner-design.md): the operational +// control that stops a deployment minting work while its state is migrated, +// plus the four admission predicates the semantics matrix is written as. +// +// The RAW constants behind it — EXECUTION_FENCE_TABLE, EXECUTION_FENCE_STATES, +// EXECUTION_FENCE_ROW_ID, EXECUTION_FENCE_DDL — are deliberately absent: they +// are the provisioning protocol's, shipped on +// `@proofoftech/flowsafe/deployment-identity-protocol` where the provisioning +// CLI and fleet-control can reach them too. Publishing them twice would let a +// consumer pin the table name from one subpath and the DDL from the other and +// never learn they had drifted. +export type { + ExecutionFenceDatabase, + ExecutionFenceReading, + ExecutionFenceRefusal, + ExecutionFenceState, + ExecutionFenceStatement, + ExecutionFenceStoreOptions, + ExecutionFenceTransition, + ExecutionFenceWiring, + // One arm of ExecutionFenceRefusal, published because that union is: a + // consumer that catches a fence refusal on the far side of a Durable Object + // boundary is handed THIS shape, and a union arm it cannot name is a surface + // it cannot write a handler's type against. + WireExecutionFenceRefusal, +} from './execution-fence.js'; +export { + admitsDrainableExecution, + admitsExistingRun, + admitsRunStart, + admitsWorkAuthoring, + assertExecutionFenceState, + ExecutionFencedError, + ExecutionFenceStore, + ExecutionFenceUnreadableError, + executionFencedResponse, + // The one memo every host composes its fence through — see executionFenceFor. + executionFenceFor, + executionFenceReadingPayload, + FenceTransitionConflictError, + InvalidExecutionFenceRequestError, + isExecutionFenceRefusal, + // OPEN_EXECUTION_FENCE is deliberately NOT exported: `readExecutionFence` + // is the only supported way to resolve an absent fence, so no consumer can + // hand-roll a ternary that gets the open case subtly wrong. + // EXECUTION_FENCE_SUSPEND_KEY is deliberately NOT exported here; it is + // published only from `./background-tasks`, whose host stamps and reads it. + readExecutionFence, +} from './execution-fence.js'; export { EXECUTION_PRINCIPAL_HEADER } from './execution-principal-header.js'; export type { HubStreamEvent, PresenceMember } from './hub-do.js'; export { HUB_INSTANCE_NAME, HubDurableObject } from './hub-do.js'; @@ -92,8 +144,41 @@ export type { InitOptions, InitResult, InitSource, + StorageInitOptions, } from './init.js'; export { init } from './init.js'; +// The drain inventory: the read-only surface an operator proves a deployment +// empty with, and the table census that keeps that proof complete as new +// tables arrive. +export type { + DeploymentInventoryOptions, + DrainProofContract, + FlowsafeTableEntry, + InventoryCategory, + InventoryCategoryClass, + InventoryCategoryDescriptor, + InventoryDatabase, + InventoryEntry, + InventoryIndex, + InventoryPage, + InventoryReadOptions, + InventoryStatement, + InventoryTableAccounting, + UnenumerableState, +} from './inventory.js'; +export { + DeploymentInventory, + FLOWSAFE_TABLES, + INVENTORY_CATEGORIES, + INVENTORY_CATEGORY_DESCRIPTORS, + INVENTORY_DEFAULT_LIMIT, + INVENTORY_DRAIN_PROOF, + INVENTORY_INDEX, + INVENTORY_MAX_LIMIT, + INVENTORY_UNENUMERABLE, + InvalidInventoryRequestError, + isInventoryCategory, +} from './inventory.js'; // Agent-memory id chokepoint: mint server-owned thread ids and validate trusted // host business keys used as resource ids (clients never supply either). export { mintThreadId, resourceIdFromKey } from './memory-id.js'; @@ -141,6 +226,59 @@ export type { ScheduleStartTarget, } from './schedule-source.js'; export { resolveScheduleStartOwner } from './schedule-source.js'; +// Owner-bound idempotent start (docs/do-runner-design.md): the reservation that +// makes a retried start converge onto the run it already made instead of paying +// for a second one. Its eight structured reasons are five decision refusals — +// IDEMPOTENT_START_OWNER_MISMATCH (403), IDEMPOTENT_START_TARGET_MISMATCH +// (409), IDEMPOTENT_START_PENDING (503), IDEMPOTENT_START_UNRESOLVABLE (409), +// and IDEMPOTENT_START_ALREADY_SETTLED (409) — plus +// IDEMPOTENT_START_UNSUPPORTED (503), INVALID_START_IDEMPOTENCY_REQUEST (400), +// and IDEMPOTENT_START_UNREADABLE (503). +// +// START_IDEMPOTENCY_DDL and the two index statements are exported for the drain +// inventory and for a host that owns its own migrations; the table NAME rides +// with them because — unlike the fence's — this table is created by the store +// itself, so there is only ever one definition of it to import. +export type { + IdempotentStartDecision, + IdempotentStartSurface, + StartIdempotencyDatabase, + StartIdempotencyStatement, + StartIdempotencyStoreOptions, + StartIdempotencyWiring, + StartReservation, + StartReservationOutcome, + StartReservationOwner, + StartReservationRefusal, + StartReservationRequest, + StartReservationState, + StartTargetKind, +} from './start-idempotency.js'; +export { + beginIdempotentStart, + IdempotentStartAlreadySettledError, + IdempotentStartPendingError, + IdempotentStartUnresolvableError, + InvalidStartIdempotencyRequestError, + isStartReservationRefusal, + requireStartIdempotency, + rollbackFencedStart, + START_IDEMPOTENCY_DDL, + START_IDEMPOTENCY_RUN_INDEX_DDL, + START_IDEMPOTENCY_STATE_INDEX_DDL, + START_IDEMPOTENCY_TABLE, + START_RESERVATION_STATES, + START_TARGET_KINDS, + StartIdempotencyStore, + StartIdempotencyUnsupportedError, + StartReservationOwnerMismatchError, + StartReservationTargetMismatchError, + StartReservationUnreadableError, + // The one memo every host composes its reservation store through — same + // reasoning as executionFenceFor: two stores over two bindings are two tables + // answering the same key. + startIdempotencyFor, +} from './start-idempotency.js'; // Per-suspension deadlines: the reserved suspend-payload key that arms one, the // timeout envelope a resumed step branches on, and the bounds each is validated // against (docs/do-runner-design.md, "Per-suspension deadlines"). The stored diff --git a/packages/flowsafe/src/do-runner/init.ts b/packages/flowsafe/src/do-runner/init.ts index c22b19a..4ea2069 100644 --- a/packages/flowsafe/src/do-runner/init.ts +++ b/packages/flowsafe/src/do-runner/init.ts @@ -19,9 +19,20 @@ import { import type { D1DatabaseBinding } from './cf-types.js'; import { createD1Storage } from './d1-storage.js'; +import type { + ExecutionFenceDatabase, + ExecutionFenceWiring, +} from './execution-fence.js'; +import { ExecutionFenceStore } from './execution-fence.js'; import type { HostPubSub } from './pubsub.js'; import type { RequestContextProvider } from './runtime.js'; import { RunnerRuntime } from './runtime.js'; +import type { + StartIdempotencyDatabase, + StartIdempotencyStore, + StartIdempotencyWiring, +} from './start-idempotency.js'; +import { startIdempotencyFor } from './start-idempotency.js'; /** Workers env shape init() understands directly. */ export interface DORunnerEnv { @@ -54,8 +65,92 @@ export interface InitOptions { * stays the fallback). */ pubsub?: HostPubSub; + /** + * The deployment execution fence (do-runner/execution-fence.ts). + * + * OPTIONAL only for a `{ DB }` source, where init builds one from that same + * binding — the fence must live in the database it fences, and init is the + * one place that holds both. Passing a store here overrides the auto-build + * (a host sharing one instance across its Durable Objects). There is + * deliberately NO `'none'` on this branch: a host that hands init a database + * cannot end up with a fence-less runtime, whatever it forgets or opts out + * of, which is the fail-closed-by-construction half of the contract. + * + * REQUIRED, and widened to include the opt-out, for a `{ storage }` source — + * see ExecutionFenceWiring, which spells out why every fence option that CAN + * be required is. + */ + executionFence?: ExecutionFenceStore; + /** + * The deployment's start reservations, for a `{ storage }` source only. + * + * A `{ DB }` source ignores this and always gets the store built from its own + * binding — the reservation table must live in the database the runs live in, + * and init is the one place that holds both. + * + * OPTIONAL on THIS branch only, where it is ignored. `StorageInitOptions` + * requires it, and widens it to include the opt-out — see there for why a + * `{ storage }` host must write the answer down. + */ + startIdempotency?: StartIdempotencyWiring; } +/** + * The reservation store this runtime settles against. + * + * A `{ DB }` source gets one built from its own binding, with no option and no + * opt-out — the reservation table must live in the database the runs live in, + * and init is the one place that holds both, so there is no third answer for a + * host to get wrong. A `{ storage }` source has no binding to derive one from, + * so it gets whatever it was handed and `undefined` otherwise. + * + * `startIdempotencyFor` is the per-binding memo, so the store the runtime + * settles against and the store the run router reserved into are the same + * object whenever both were built from the same `env.DB`. + */ +function startIdempotencyForSource( + source: InitSource, + configured: StartIdempotencyWiring | undefined, +): StartIdempotencyStore | undefined { + if ('storage' in source) { + return configured === undefined || configured === 'none' + ? undefined + : configured; + } + // The same boundary widening createD1Storage makes on this identical value: + // D1DatabaseBinding types `prepare` as returning `unknown` so the shared env + // shape needs no statement type. + return startIdempotencyFor(source.DB as unknown as StartIdempotencyDatabase); +} + +/** + * InitOptions for a `{ storage }` source: the fence wiring AND the reservation + * wiring are mandatory, and both are widened to admit the opt-out. Written as + * an intersection rather than an `extends`, because a subtype may not WIDEN an + * inherited property's type — and the widening is the point: `'none'` exists + * only on this branch. + * + * `startIdempotency` is required HERE and optional on `InitOptions` because the + * two branches fail differently. A `{ DB }` host cannot get this wrong: init + * builds the store from the binding, and `DurableObjectRunner.build` refuses to + * serve a runtime that lacks one while a DB binding is present. A `{ storage }` + * host has no binding to derive one from, and its failure is SILENT and split: + * such a host can still wire a real store into its run router (the router takes + * its own), so keys reserve and claim normally — and then nothing ever settles + * them, because the runtime that sees every terminal transition was never given + * the store. The reservation stays `started` forever, and the next retry of + * that key is told UNRESOLVABLE instead of replaying a run that completed + * perfectly well. Making the host WRITE `'none'` turns that into a decision + * someone made; nothing about an omission could have said it. + */ +export type StorageInitOptions = Omit< + InitOptions, + 'executionFence' | 'startIdempotency' +> & { + executionFence: ExecutionFenceWiring; + startIdempotency: StartIdempotencyWiring; +}; + export interface InitResult { createWorkflow: typeof coreCreateWorkflow; createStep: typeof createStep; @@ -66,13 +161,47 @@ export interface InitResult { * which is what keeps it single per DO. */ pubsub?: HostPubSub; + /** + * The deployment execution fence, or undefined for an explicitly unfenced + * host. THE accessor for it, for the same reason as `pubsub`: a route that + * built its own store could be gating a different database than the runtime + * it sits in front of. Thread-DO signal routes read it off `scope.init`. + */ + executionFence?: ExecutionFenceStore; + /** + * The deployment's start reservations, or undefined for a `{ storage }` host + * with no database to reserve against. THE accessor for it, for the same + * reason as `executionFence`: a surface that built its own store could be + * reserving into a different database than the runtime that settles. + */ + startIdempotency?: StartIdempotencyStore; } +export function init(source: DORunnerEnv, options?: InitOptions): InitResult; +export function init( + source: { storage: MastraCompositeStore }, + options: StorageInitOptions, +): InitResult; +/** + * A source whose shape is only known at runtime (a host that accepts either). + * The wiring is REQUIRED here for the same reason it is on the `{ storage }` + * branch: the compiler cannot tell which branch this call will take, so it + * cannot know whether init would have built a fence. `'none'` still applies + * only to a `{ storage }` source — a `{ DB }` one is fenced regardless, so + * choosing the opt-out cannot leave a database-backed runtime unfenced. + */ +export function init( + source: InitSource, + options: StorageInitOptions, +): InitResult; export function init( source: InitSource, - options: InitOptions = {}, + options: Omit & { + executionFence?: ExecutionFenceWiring; + } = {}, ): InitResult { let storage: MastraCompositeStore; + let executionFence: ExecutionFenceStore | undefined; if ('storage' in source) { if (options.id !== undefined || options.tablePrefix !== undefined) { // Silently ignoring these would mask a misconfiguration: they only @@ -82,22 +211,53 @@ export function init( ); } storage = source.storage; + executionFence = + options.executionFence === 'none' ? undefined : options.executionFence; } else { storage = createD1Storage({ binding: source.DB, id: options.id, tablePrefix: options.tablePrefix, }); + // Fail-closed by construction: a host that hands init a database gets a + // fenced runtime whether or not it remembered to ask for one, and the + // option's type on this branch admits no opt-out. The cast is the same + // boundary widening createD1Storage makes on the identical value + // (d1-storage.ts) — D1DatabaseBinding deliberately types `prepare` as + // returning `unknown` so the shared env shape needs no statement type. + // The overload for this source types `executionFence` as a store, so the + // opt-out cannot be written here; the narrowing is what makes that visible + // to the implementation signature, which sees both branches' options. + const configured = options.executionFence; + executionFence = + configured === undefined || configured === 'none' + ? new ExecutionFenceStore( + source.DB as unknown as ExecutionFenceDatabase, + ) + : configured; } + const startIdempotency = startIdempotencyForSource( + source, + options.startIdempotency, + ); const runtime = new RunnerRuntime({ storage, requestContextForRun: options.requestContextForRun, // Threaded directly because every DO subclass // returns THIS runtime from build(), so a host that configures a pubsub - // reaches the runtime's createRun sites with no host change (Track A wires - // those). Handing it only to InitResult would strand it — build() returns a - // RunnerRuntime, not an InitResult, so the run-DO path would drop it. + // reaches the runtime's createRun sites with no host change (the durable + // agent wires those). Handing it only to InitResult would strand it: + // build() returns a RunnerRuntime, not an InitResult, so the run-DO path + // would drop it. pubsub: options.pubsub, + // Same reasoning, and the reason the fence is a construction-time argument + // rather than a per-call one: the runtime IS the closure guarantee, so it + // must not be possible to reach start()/resume() with the fence left off. + executionFence, + // Same construction-time reasoning: the runtime is the one layer that sees + // every terminal transition, so it must not be possible to build one that + // executes runs but cannot mark their reservations spent. + startIdempotency, }); // Cast preserves core's generic call-site inference (6 type params); the @@ -115,5 +275,7 @@ export function init( createStep, runtime, pubsub: options.pubsub, + executionFence, + startIdempotency, }; } diff --git a/packages/flowsafe/src/do-runner/inventory.test.ts b/packages/flowsafe/src/do-runner/inventory.test.ts new file mode 100644 index 0000000..fef5534 --- /dev/null +++ b/packages/flowsafe/src/do-runner/inventory.test.ts @@ -0,0 +1,1045 @@ +// SPDX-License-Identifier: Apache-2.0 +// The drain inventory, over the REAL schemas. +// +// Every table below is created by the production code that owns it — +// createD1Storage plus the signal and schedule domains, the approval store +// factory, the resource-ownership schema, the reservation store, the +// subscription factory. A fixture that hand-wrote the DDL would pass forever +// while a column rename quietly emptied a category, which is the one failure +// this surface must not have: an empty category is what an operator reads as +// permission to migrate. + +import { describe, expect, it } from 'vitest'; +import { z } from 'zod'; + +import { + openSqlite, + type SqliteDatabase, + sqliteUnitDatabase, +} from '../../test-support/sqlite.js'; +import type { ApprovalRecord } from '../approval-api/index.js'; +import { + createResourceOwnershipSchema, + D1ApprovalStoreFactory, + D1ResourceOwnershipStore, +} from '../approval-api/index.js'; +// From the leaf that owns it, not the barrel: the table name is deliberately +// NOT part of the package's public surface — it is the one home two layers +// share, and widening it to consumers would invite queries the store cannot +// keep correct. +import { APPROVALS_TABLE } from '../approval-api/types.js'; +// The public marker the background-task host stamps. The inventory imports its +// single home from do-runner, while this fixture imports the unchanged public +// background-tasks surface. +import { EXECUTION_FENCE_SUSPEND_KEY } from '../background-tasks/index.js'; +import { createScheduleStorageDomains } from '../schedules/storage.js'; +import { D1SubscriptionStoreFactory } from '../signal-providers/index.js'; +import { createSignalStorageDomains } from '../signals/storage.js'; +import { createD1Storage, RESOURCE_OWNER_TABLE } from './d1-storage.js'; +import { RUN_OWNER_RECOVERY_DELAY_MS } from './durable-object.js'; +import { + DEPLOYMENT_IDENTITY_HEADER, + DurableObjectRunner, + type DurableObjectRunOwnershipStore, + EXECUTION_PRINCIPAL_HEADER, + ExecutionFenceStore, + type RunnerRuntime, +} from './index.js'; +import { init } from './init.js'; +import { + D1_MAX_BOUND_PARAMETERS, + DeploymentInventory, + INVENTORY_CATEGORY_DESCRIPTORS, + INVENTORY_DRAIN_PROOF, + INVENTORY_UNENUMERABLE, + InvalidInventoryRequestError, + type InventoryCategory, + type InventoryDatabase, + RUN_OWNER_FIXED_BINDINGS, + RUN_OWNER_LOOKUP_CHUNK, +} from './inventory.js'; +import { + START_IDEMPOTENCY_TABLE, + StartIdempotencyStore, +} from './start-idempotency.js'; + +const NOW = Date.parse('2026-08-24T12:00:00.000Z'); + +/** Every category name, so a sweep in a test can never miss one. */ +const ALL_CATEGORIES: readonly InventoryCategory[] = + INVENTORY_CATEGORY_DESCRIPTORS.map((entry) => entry.category); + +const WORK_CATEGORIES: readonly InventoryCategory[] = + INVENTORY_CATEGORY_DESCRIPTORS.filter((entry) => entry.class === 'work').map( + (entry) => entry.category, + ); + +/** + * A database that RECORDS every statement prepared through it. + * + * This is the read-only pin's primary instrument, and it is stronger than + * counting rows afterwards: a write that happened to change nothing (a + * `CREATE TABLE IF NOT EXISTS` on an existing table, an UPDATE matching zero + * rows) leaves the data identical and is still exactly the class of side effect + * this surface must not have. What the deployment is about to be COPIED from + * must not be mutated by the act of measuring it. + */ +function recordingDatabase(binding: unknown): { + db: InventoryDatabase; + statements: string[]; + /** One entry per `bind()`: the SQL and how many parameters it carried. */ + bindings: Array<{ sql: string; count: number }>; +} { + const statements: string[] = []; + const bindings: Array<{ sql: string; count: number }> = []; + const inner = binding as InventoryDatabase; + return { + statements, + bindings, + db: { + prepare(query: string) { + statements.push(query); + const prepared = inner.prepare(query); + return { + ...prepared, + bind(...values: unknown[]) { + bindings.push({ sql: query, count: values.length }); + return prepared.bind(...values); + }, + }; + }, + }, + }; +} + +/** The schema exactly as SQLite records it — the evidence no DDL ran. */ +function schemaSnapshot(sqlite: SqliteDatabase): unknown[] { + return sqlite + .prepare( + 'SELECT type, name, tbl_name, sql FROM sqlite_master ORDER BY type, name', + ) + .all(); +} + +/** Every row of every table, so a change of any kind shows up. */ +function dataSnapshot(sqlite: SqliteDatabase): Record { + const tables = ( + sqlite + .prepare( + "SELECT name FROM sqlite_master WHERE type='table' AND name NOT LIKE 'sqlite_%' ORDER BY name", + ) + .all() as Array<{ name: string }> + ).map((row) => row.name); + const snapshot: Record = {}; + for (const table of tables) { + snapshot[table] = sqlite.prepare(`SELECT * FROM ${table}`).all(); + } + return snapshot; +} + +function approval(overrides: Partial): ApprovalRecord { + const at = new Date(NOW).toISOString(); + return { + id: 'apr-1', + workflowId: 'wf', + runId: 'run-1', + title: 'approval', + connectors: [], + priority: 'normal', + status: 'pending', + createdAt: at, + updatedAt: at, + ...overrides, + }; +} + +interface Fixture { + sqlite: SqliteDatabase; + binding: unknown; + inventory: DeploymentInventory; +} + +/** + * A deployment with every table created by its real owner, and one outstanding + * item in each work category plus one row in each standing category. + */ +async function seeded(): Promise { + const sqlite = openSqlite(); + const binding = sqliteUnitDatabase(sqlite); + const storage = createD1Storage({ + binding: binding as never, + domains: { + ...createSignalStorageDomains(binding as never), + ...createScheduleStorageDomains(binding as never), + }, + }); + + // --- runs: one suspended run, minted by the real runtime ------------------ + const { createWorkflow, createStep, runtime } = init( + { storage }, + { startIdempotency: 'none', executionFence: 'none' }, + ); + const gate = createStep({ + id: 'gate', + inputSchema: z.object({}), + outputSchema: z.object({}), + suspendSchema: z.object({ reason: z.string() }), + resumeSchema: z.object({ go: z.boolean() }), + execute: async ({ resumeData, suspend }) => { + if (!resumeData) return suspend({ reason: 'wait' }); + return {}; + }, + }); + createWorkflow({ + id: 'gated', + inputSchema: z.object({}), + outputSchema: z.object({}), + }) + .then(gate) + .commit(); + await runtime.start('gated', { runId: 'abc_r1', inputData: {} }); + await storage.init(); + + // --- approvals-waiting ---------------------------------------------------- + // Distinct runIds: the store's open-step uniqueness index refuses a second + // OPEN record for the same (workflow, run, step), which is the invariant that + // stops a re-suspension filing a duplicate. + const approvals = new D1ApprovalStoreFactory(binding as never).store(); + await approvals.create(approval({ id: 'apr-open', status: 'pending' })); + await approvals.create( + approval({ id: 'apr-claimed', status: 'claimed', runId: 'run-1b' }), + ); + await approvals.create( + approval({ id: 'apr-done', status: 'approved', runId: 'run-2' }), + ); + + // --- resource-owners: one settled claim, one live reservation ------------- + await createResourceOwnershipSchema(binding as never); + const owners = new D1ResourceOwnershipStore(binding as never); + await owners.claim('run', 'abc_r1', { kind: 'human', id: 'ada' }); + await owners.reserveAll( + [{ kind: 'thread', resourceId: 'thr-1' }], + { kind: 'service', id: 'svc' }, + 'tok-1', + ); + + // --- start-reservations --------------------------------------------------- + const reservations = new StartIdempotencyStore(binding as never, { + now: () => NOW, + }); + await reservations.reserve({ + key: 'key-live', + owner: { kind: 'human', id: 'ada' }, + targetKind: 'workflow', + targetId: 'gated', + mintRunId: () => 'abc_r2', + }); + await reservations.reserve({ + key: 'key-settled', + owner: { kind: 'human', id: 'ada' }, + targetKind: 'workflow', + targetId: 'gated', + mintRunId: () => 'abc_r3', + }); + await reservations.settleRun('abc_r3'); + + // --- signal-subscriptions ------------------------------------------------- + await new D1SubscriptionStoreFactory(binding as never, { + uuid: () => 'sub-1', + }) + .store() + .subscribe({ + providerId: 'github', + externalResourceId: 'octo/repo#1', + threadId: 'thr-1', + resourceId: 'res-1', + }); + + // --- the rows whose writers are Mastra's own domains ---------------------- + // Inserted directly, into tables the production DDL above created: what this + // suite pins is the READER's predicate against the real column names, and + // driving a whole agent loop to park one background task would test the loop. + const iso = new Date(NOW).toISOString(); + sqlite + .prepare( + `INSERT INTO mastra_background_tasks + (id, tool_call_id, tool_name, agent_id, run_id, thread_id, resource_id, + status, args, result, error, suspend_payload, retry_count, + max_retries, timeout_ms, createdAt, startedAt, suspendedAt, completedAt) + VALUES (?, 'tc', 'tool', 'agent', 'abc_r1', 'thr-1', 'res-1', ?, '{}', + NULL, NULL, ?, 0, 0, 1000, ?, NULL, NULL, ?)`, + ) + .run('bg-queued', 'pending', null, iso, null); + sqlite + .prepare( + `INSERT INTO mastra_background_tasks + (id, tool_call_id, tool_name, agent_id, run_id, thread_id, resource_id, + status, args, result, error, suspend_payload, retry_count, + max_retries, timeout_ms, createdAt, startedAt, suspendedAt, completedAt) + VALUES (?, 'tc', 'tool', 'agent', 'abc_r1', 'thr-1', 'res-1', ?, '{}', + NULL, NULL, ?, 0, 0, 1000, ?, NULL, ?, NULL)`, + ) + .run( + 'bg-parked', + 'suspended', + JSON.stringify({ [EXECUTION_FENCE_SUSPEND_KEY]: { state: 'draining' } }), + iso, + iso, + ); + sqlite + .prepare( + `INSERT INTO mastra_background_tasks + (id, tool_call_id, tool_name, agent_id, run_id, thread_id, resource_id, + status, args, result, error, suspend_payload, retry_count, + max_retries, timeout_ms, createdAt, startedAt, suspendedAt, completedAt) + VALUES (?, 'tc', 'tool', 'agent', 'abc_r1', 'thr-1', 'res-1', ?, '{}', + NULL, NULL, NULL, 0, 0, 1000, ?, NULL, NULL, ?)`, + ) + .run('bg-done', 'completed', iso, iso); + + const past = new Date(NOW - 60_000).toISOString(); + const future = new Date(NOW + 3_600_000).toISOString(); + const insertNotification = ( + id: string, + status: string, + deliverAt: string | null, + ): void => { + sqlite + .prepare( + `INSERT INTO mastra_notifications + (id, thread_id, source, kind, priority, status, summary, payload, + resourceId, agentId, sourceId, dedupeKey, coalesceKey, + coalescedCount, attributes, createdAt, updatedAt, deliverAt, + summaryAt, deliveryReason, deliveryAttempts, lastDeliveryAttemptAt, + lastDeliveryError, deliveredSignalId, summarySignalId, deliveredAt, + seenAt, dismissedAt, archivedAt, discardedAt, metadata, + insertionOrdinal) + VALUES (?, 'thr-1', 'src', 'kind', 'medium', ?, 'summary', NULL, + 'res-1', 'agent', NULL, NULL, NULL, 1, NULL, ?, ?, ?, NULL, + NULL, 0, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, + NULL, NULL)`, + ) + .run(id, status, past, past, deliverAt); + }; + insertNotification('ntf-due', 'pending', past); + insertNotification('ntf-later', 'pending', future); + insertNotification('ntf-never', 'pending', null); + insertNotification('ntf-delivered', 'delivered', past); + + const insertTrigger = (id: string, outcome: string): void => { + sqlite + .prepare( + `INSERT INTO mastra_schedule_triggers + (id, scheduleId, runId, scheduledFireAt, actualFireAt, outcome, + error, triggerKind, parentTriggerId, metadata) + VALUES (?, 'sch-1', NULL, ?, ?, ?, NULL, 'schedule-fire', NULL, NULL)`, + ) + .run(id, NOW, NOW, outcome); + }; + insertTrigger('trg-deferred', 'deferred'); + insertTrigger('trg-started', 'started'); + insertTrigger('trg-skipped', 'skipped'); + + sqlite + .prepare( + `INSERT INTO mastra_schedules + (id, target, cron, timezone, status, nextFireAt, lastFireAt, lastRunId, + createdAt, updatedAt, metadata, ownerType, ownerId, creatorRole, + deletionRequestedAt) + VALUES ('sch-1', ?, '* * * * *', NULL, 'active', ?, NULL, NULL, ?, ?, + NULL, NULL, NULL, NULL, NULL)`, + ) + .run( + JSON.stringify({ type: 'workflow', workflowId: 'gated' }), + NOW + 60_000, + NOW, + NOW, + ); + + return { + sqlite, + binding, + inventory: new DeploymentInventory(binding as InventoryDatabase, { + now: () => NOW, + }), + }; +} + +/** Page a category to exhaustion, following its own cursors. */ +async function drain( + inventory: DeploymentInventory, + category: InventoryCategory, + limit?: number, +): Promise { + const keys: string[] = []; + let cursor: string | undefined; + for (let pass = 0; pass < 100; pass += 1) { + const page = await inventory.read(category, { + ...(cursor === undefined ? {} : { cursor }), + ...(limit === undefined ? {} : { limit }), + }); + keys.push(...page.entries.map((entry) => JSON.stringify(entry.key))); + if (page.cursor === undefined) return keys; + cursor = page.cursor; + } + throw new Error(`inventory paging did not terminate for ${category}`); +} + +describe('deployment drain inventory', () => { + it('reports the index as a contract: every category, its class, what it cannot see, and how to read empty', async () => { + // #given — an inventory over any database (the index is a constant: what + // can be asked is not a function of what happens to be stored). + const { inventory } = await seeded(); + + // #when + const index = inventory.index(); + + // #then — work first, then standing, and each named exactly once. + expect(index.categories.map((entry) => entry.category)).toEqual([ + ...WORK_CATEGORIES, + 'schedules', + 'signal-subscriptions', + ]); + // #then — the two states no query can see are DECLARED, not omitted. + expect(index.unenumerable.map((entry) => entry.name)).toEqual([ + 'run-owner-recovery-journal', + 'persisted-idle-signals', + ]); + // #then — and the rule an empty answer means something under. + expect(index.drainProof.reachableFrom).toEqual(['draining']); + expect(index.drainProof.reading).toMatch(/point-in-time observation/); + expect(index.drainProof.proof).toMatch(/TWO consecutive full sweeps/); + const recoveryCadence = `${RUN_OWNER_RECOVERY_DELAY_MS.toLocaleString( + 'en-US', + )} ms`; + expect(INVENTORY_UNENUMERABLE[0]?.bound).toContain(recoveryCadence); + expect(INVENTORY_DRAIN_PROOF.proof).toContain(recoveryCadence); + }); + + it('reads each work category with the predicate its production writer settles on', async () => { + // #given — one outstanding item per category beside settled siblings that + // must NOT be reported: a decided approval, a spent reservation, a + // completed background task, a delivered notification, settled fire + // history, and a released ownership row. + const { inventory } = await seeded(); + + // #when / #then — runs: the suspended run, annotated with its owner. + const runs = await inventory.read('runs'); + expect(runs.entries).toEqual([ + { + key: ['gated', 'abc_r1'], + detail: expect.objectContaining({ + status: 'suspended', + owner: 'human:ada', + }), + }, + ]); + expect(runs.count).toBe(1); + + // #then — approvals: pending AND claimed (both still undecided); never the + // approved one. + expect((await inventory.read('approvals-waiting')).entries).toEqual([ + { + key: ['apr-claimed'], + detail: expect.objectContaining({ status: 'claimed' }), + }, + { + key: ['apr-open'], + detail: expect.objectContaining({ status: 'pending' }), + }, + ]); + + // #then — deferred dispatches only: the started and skipped fires are + // settled history in the same table, and counting them would keep this + // category permanently non-empty. + expect( + (await inventory.read('schedule-deferred-dispatches')).entries.map( + (entry) => entry.key, + ), + ).toEqual([['trg-deferred']]); + + // #then — notifications: only the DUE pending row is drainable work; the + // future-dated and the never-due rows are reported as a total instead. + const notifications = await inventory.read('pending-notifications'); + expect(notifications.entries.map((entry) => entry.key)).toEqual([ + ['thr-1', 'ntf-due'], + ]); + // `count` is this category's own rows (the DUE one); `notDue` describes + // the pending rows the page deliberately excludes. + expect(notifications.count).toBe(1); + expect(notifications.totals).toEqual({ notDue: 2 }); + + // #then — background tasks: nonterminal only, and the fence-parked one is + // flagged so an operator can tell it from one awaiting a webhook. + const tasks = await inventory.read('background-tasks'); + expect( + tasks.entries.map((entry) => [entry.key[0], entry.detail.fenceSuspended]), + ).toEqual([ + ['bg-parked', true], + ['bg-queued', false], + ]); + expect(tasks.totals).toEqual({ fenceSuspended: 1 }); + + // #then — ownership: the unsettled reservation, never the settled claim, + // and never the reservation token itself. + const ownersPage = await inventory.read('resource-owners'); + expect(ownersPage.entries).toEqual([ + { + key: ['thread', 'thr-1'], + detail: { owner_kind: 'service', owner_id: 'svc' }, + }, + ]); + + // #then — reservations: reserved/started, never terminal. + expect( + (await inventory.read('start-reservations')).entries.map( + (entry) => entry.key, + ), + ).toEqual([['key-live']]); + }); + + it('reports standing configuration without asking a drain to empty it', async () => { + // #given + const { inventory } = await seeded(); + + // #when + const schedules = await inventory.read('schedules'); + const subscriptions = await inventory.read('signal-subscriptions'); + + // #then — the schedule's trigger identity comes through, so a reconciling + // operator sees WHAT it will fire, not just that it exists. + expect(schedules.class).toBe('standing'); + expect(schedules.entries).toEqual([ + { + key: ['sch-1'], + detail: expect.objectContaining({ + status: 'active', + targetType: 'workflow', + targetId: 'gated', + }), + }, + ]); + expect(subscriptions.class).toBe('standing'); + expect(subscriptions.entries).toEqual([ + { + key: ['sub-1'], + detail: expect.objectContaining({ + provider_id: 'github', + thread_id: 'thr-1', + }), + }, + ]); + }); + + it('counts a snapshot it cannot classify as WORK rather than as finished', async () => { + // #given — a snapshot row whose JSON is corrupt, beside one whose JSON is + // valid but carries no status. Neither can be proven terminal. + const { sqlite, inventory } = await seeded(); + const iso = new Date(NOW).toISOString(); + sqlite + .prepare( + `INSERT INTO mastra_workflow_snapshot + (workflow_name, run_id, resourceId, snapshot, createdAt, updatedAt) + VALUES (?, ?, NULL, ?, ?, ?)`, + ) + .run('gated', 'abc_corrupt', '{not json', iso, iso); + sqlite + .prepare( + `INSERT INTO mastra_workflow_snapshot + (workflow_name, run_id, resourceId, snapshot, createdAt, updatedAt) + VALUES (?, ?, NULL, ?, ?, ?)`, + ) + .run('gated', 'abc_nostatus', '{"steps":{}}', iso, iso); + + // #when + const runs = await inventory.read('runs'); + + // #then — both are reported. A row nobody can prove is finished is exactly + // the row a migration must not walk away from, and the alternative reading + // would let a corrupt snapshot certify a deployment empty. + expect(runs.entries.map((entry) => entry.key[1])).toEqual([ + 'abc_corrupt', + 'abc_nostatus', + 'abc_r1', + ]); + }); + + it('excludes a run whose terminal status still owes lifecycle cleanup', async () => { + // #given — a 'timed_out' run with no cleanupCompletedAt (still compensating) + // beside one that has completed cleanup. The retention purge draws the line + // here; so must the drain proof, or the two disagree about what "finished" + // means on the same deployment. + const { sqlite, inventory } = await seeded(); + const iso = new Date(NOW).toISOString(); + const snapshot = (cleanup: string | null): string => + JSON.stringify({ + status: 'timed_out', + requestContext: { + 'flowsafe.runLifecycle': + cleanup === null + ? {} + : { terminal: { cleanupCompletedAt: cleanup } }, + }, + }); + sqlite + .prepare( + `INSERT INTO mastra_workflow_snapshot + (workflow_name, run_id, resourceId, snapshot, createdAt, updatedAt) + VALUES (?, ?, NULL, ?, ?, ?)`, + ) + .run('gated', 'abc_cleaning', snapshot(null), iso, iso); + sqlite + .prepare( + `INSERT INTO mastra_workflow_snapshot + (workflow_name, run_id, resourceId, snapshot, createdAt, updatedAt) + VALUES (?, ?, NULL, ?, ?, ?)`, + ) + .run('gated', 'abc_cleaned', snapshot(iso), iso, iso); + + // #when + const runs = await inventory.read('runs'); + + // #then — mid-cleanup is work; cleanup-complete is not. + expect(runs.entries.map((entry) => entry.key[1])).toEqual([ + 'abc_cleaning', + 'abc_r1', + ]); + }); + + it('reads a table that was never created as an EMPTY category, not a fault', async () => { + // #given — a database with nothing in it at all: the state of a deployment + // provisioned and never used. Every table here is created lazily by the + // first feature that writes it. + const sqlite = openSqlite(); + const inventory = new DeploymentInventory( + sqliteUnitDatabase(sqlite) as InventoryDatabase, + { now: () => NOW }, + ); + + // #when — the whole sweep an operator would run. + const pages = await inventory.sweep(); + + // #then — every category answers, none throws, and no count is invented + // for a table that does not exist. + expect(pages.map((page) => page.category)).toEqual(ALL_CATEGORIES); + expect(pages.every((page) => page.entries.length === 0)).toBe(true); + expect(pages.every((page) => page.cursor === undefined)).toBe(true); + expect(pages.every((page) => page.count === undefined)).toBe(true); + }); + + it('still reports runs when the ownership registry does not exist', async () => { + // #given — a deployment whose runs exist but that never wired resource + // ownership. The owner is an ANNOTATION, and losing an annotation must + // never cost the category a drain proof depends on. + const { sqlite, inventory } = await seeded(); + sqlite.exec(`DROP TABLE ${RESOURCE_OWNER_TABLE}`); + + // #when + const runs = await inventory.read('runs'); + + // #then — the run is still there, just unannotated. + expect(runs.entries.map((entry) => entry.key[1])).toEqual(['abc_r1']); + expect(runs.entries[0]?.detail.owner).toBeUndefined(); + }); + + it('pages a keyset that neither skips a row nor returns one twice', async () => { + // #given — more approvals than one page holds. + const { binding, inventory } = await seeded(); + const approvals = new D1ApprovalStoreFactory(binding as never).store(); + for (let index = 0; index < 7; index += 1) { + await approvals.create( + approval({ id: `apr-p${index}`, runId: `run-p${index}` }), + ); + } + + // #when — paged two at a time, following the inventory's own cursors. + const paged = await drain(inventory, 'approvals-waiting', 2); + + // #then — identical to the single-page read, in the same order, with no + // duplicates. A keyset that drifted would either strand rows (a drain that + // never proves empty) or repeat them (a proof that never converges). + const whole = await drain(inventory, 'approvals-waiting'); + expect(paged).toEqual(whole); + expect(new Set(paged).size).toBe(paged.length); + expect(paged.length).toBe(9); + }); + + it('offers a cursor only while a page is full, so an empty page IS the end', async () => { + // #given — exactly the drain proof's observation: a work category whose + // page did not fill. + const { inventory } = await seeded(); + + // #when + const page = await inventory.read('runs', { limit: 50 }); + + // #then — no cursor. A cursor here would make an operator page again and + // read the same empty answer forever. + expect(page.cursor).toBeUndefined(); + expect(page.count).toBe(1); + }); + + it('counts only on the first page of a sweep', async () => { + // #given + const { inventory } = await seeded(); + + // #when — a continuation page. + const first = await inventory.read('approvals-waiting', { limit: 1 }); + const second = await inventory.read('approvals-waiting', { + limit: 1, + ...(first.cursor === undefined ? {} : { cursor: first.cursor }), + }); + + // #then — the total is taken once, where a sweep starts; a caller already + // paging has committed to walking the category. + expect(first.count).toBe(2); + expect(second.count).toBeUndefined(); + }); + + it('refuses a malformed cursor instead of silently restarting the scan', async () => { + // #given + const { inventory } = await seeded(); + + // #then — a cursor that is not JSON, and one shaped for a DIFFERENT + // category, are both refused. Silently restarting would make a sweep + // re-read rows it had counted and never reach the end, and an operator + // waiting for two empty sweeps would wait forever without being told why. + await expect( + inventory.read('runs', { cursor: 'not-json' }), + ).rejects.toBeInstanceOf(InvalidInventoryRequestError); + await expect( + inventory.read('runs', { cursor: '["only-one"]' }), + ).rejects.toBeInstanceOf(InvalidInventoryRequestError); + await expect( + inventory.read('approvals-waiting', { cursor: '[1]' }), + ).rejects.toBeInstanceOf(InvalidInventoryRequestError); + await expect(inventory.read('runs', { limit: 0 })).rejects.toBeInstanceOf( + InvalidInventoryRequestError, + ); + }); + + it('applies the storage prefix to Mastra tables and never to the flowsafe registries', async () => { + // #given — a prefixed deployment. createD1Storage prefixes its own tables; + // the approval, ownership, reservation, and subscription registries are + // never prefixed, exactly as their production writers spell them. + const sqlite = openSqlite(); + const binding = sqliteUnitDatabase(sqlite); + const { statements, db } = recordingDatabase(binding); + const inventory = new DeploymentInventory(db, { + tablePrefix: 'p_', + now: () => NOW, + }); + + // #when + await inventory.sweep(); + + // #then — the prefix lands on exactly the Mastra-owned tables. + expect( + statements.some((sql) => sql.includes('FROM p_mastra_workflow_snapshot')), + ).toBe(true); + expect( + statements.some((sql) => sql.includes('FROM p_mastra_notifications')), + ).toBe(true); + expect( + statements.some((sql) => sql.includes(`FROM p_${APPROVALS_TABLE}`)), + ).toBe(false); + expect( + statements.some((sql) => sql.includes(`FROM ${APPROVALS_TABLE}`)), + ).toBe(true); + expect( + statements.some((sql) => sql.includes(`FROM ${START_IDEMPOTENCY_TABLE}`)), + ).toBe(true); + }); + + it('READ-ONLY PIN: a full sweep, paged to exhaustion, prepares nothing but SELECTs and changes nothing', async () => { + // #given — a populated deployment, and a complete record of it. This is the + // property the whole surface rests on: an operator runs this against the + // database they are about to copy, so measuring it must not change it. + // + // No Durable Object appears anywhere in this test, and that is the pin for + // "no DO storage key changes": the inventory's only seam is a database + // (`InventoryDatabase`, which has no `run()` at all), so there is no object + // for it to wake, no alarm for it to re-arm, and no key for it to write. + const { sqlite, binding } = await seeded(); + const schemaBefore = schemaSnapshot(sqlite); + const dataBefore = dataSnapshot(sqlite); + const { statements, db } = recordingDatabase(binding); + const inventory = new DeploymentInventory(db, { now: () => NOW }); + + // #when — the index plus every category paged to the end, one row at a + // time so every keyset continuation is exercised too. + inventory.index(); + for (const category of ALL_CATEGORIES) { + await drain(inventory, category, 1); + } + + // #then — every statement it prepared was a SELECT. Not "no rows changed": + // a CREATE TABLE IF NOT EXISTS on an existing table changes no rows either, + // and is exactly the lazy-schema write this surface must not perform. + expect(statements.length).toBeGreaterThan(ALL_CATEGORIES.length); + expect(statements.filter((sql) => !/^\s*SELECT\b/i.test(sql))).toEqual([]); + + // #then — and the database is byte-identical: same schema objects, same + // rows, same timestamps. + expect(schemaSnapshot(sqlite)).toEqual(schemaBefore); + expect(dataSnapshot(sqlite)).toEqual(dataBefore); + }); + + it('READ-ONLY PIN: an empty deployment is not given schema by being measured', async () => { + // #given — the case a lazy `#ensureSchema` would quietly break: a database + // with no tables at all. Every domain store in this package creates its + // schema on first use, so a reader built on one of them would leave a + // freshly provisioned deployment with tables it never had. + const sqlite = openSqlite(); + const { statements, db } = recordingDatabase(sqliteUnitDatabase(sqlite)); + + // #when + await new DeploymentInventory(db, { now: () => NOW }).sweep(); + + // #then — nothing was created. + expect( + sqlite.prepare("SELECT name FROM sqlite_master WHERE type='table'").all(), + ).toEqual([]); + expect(statements.filter((sql) => !/^\s*SELECT\b/i.test(sql))).toEqual([]); + }); + + it('keeps every statement inside D1 100-parameter budget on a maximum page', async () => { + // #given — a page at INVENTORY_MAX_LIMIT, which is a real knob an operator + // may turn. The owner annotation binds one parameter per row, so a 200-row + // page would build a 200-parameter IN list — which real D1 REFUSES and + // node:sqlite (32766 variables) accepts without complaint. That pairing is + // the dangerous one: the suite stays green while the route answers a + // generic 500 on the deployment an operator is trying to drain. + const { sqlite, binding } = await seeded(); + const iso = new Date(NOW).toISOString(); + const snapshot = JSON.stringify({ status: 'suspended' }); + for (let index = 0; index < 199; index += 1) { + const runId = `abc_p${String(index).padStart(3, '0')}`; + sqlite + .prepare( + `INSERT INTO mastra_workflow_snapshot + (workflow_name, run_id, resourceId, snapshot, createdAt, updatedAt) + VALUES ('gated', ?, NULL, ?, ?, ?)`, + ) + .run(runId, snapshot, iso, iso); + sqlite + .prepare( + `INSERT INTO ${RESOURCE_OWNER_TABLE} + (resource_kind, resource_id, owner_kind, owner_id, reservation_token) + VALUES ('run', ?, 'human', 'ada', NULL)`, + ) + .run(runId); + } + const { bindings, db } = recordingDatabase(binding); + + // #when — the largest page the route will serve. + const page = await new DeploymentInventory(db, { now: () => NOW }).read( + 'runs', + { limit: 200 }, + ); + + // #then — a full page, every row annotated: the chunking must not cost the + // annotation, or "fixed" would mean "stopped looking". + expect(page.entries).toHaveLength(200); + expect( + page.entries.filter((entry) => typeof entry.detail.owner !== 'string'), + ).toEqual([]); + + // #then — and no statement exceeded the budget. Asserted over EVERY + // statement rather than just the IN list, so a future reader that starts + // binding per-row fails here too. + expect(bindings.filter((entry) => entry.count > 100)).toEqual([]); + + // #then — each lookup uses the derived chunk, and that chunk stays + // equal to the D1 parameter cap less the statement's fixed bindings. + const ownerLookups = bindings.filter( + (entry) => + entry.sql.includes(RESOURCE_OWNER_TABLE) && entry.sql.includes('IN ('), + ); + expect(ownerLookups.map((entry) => entry.count)).toEqual([ + RUN_OWNER_LOOKUP_CHUNK, + RUN_OWNER_LOOKUP_CHUNK, + ]); + expect(RUN_OWNER_LOOKUP_CHUNK).toBe( + D1_MAX_BOUND_PARAMETERS - RUN_OWNER_FIXED_BINDINGS, + ); + }); + + it('OWNERSHIP-ORDERING PIN: an in-flight run with no snapshot is visible under resource-owners', async () => { + // #given — THE cross-module invariant the whole drain proof rests on, and + // the one nothing else enforces: the run object RESERVES ownership before + // it calls runtime.start (durable-object.ts #reserveRunOwner precedes + // runtime.start) and SETTLES that reservation only AFTER a summary has + // persisted (#settleRunOwnerBestEffort follows it). + // + // The engine writes a `running` snapshot inside runtime.start, so an + // executing run is normally under `runs` too — but that write lands after + // the reservation, and the unsettled reservation is what marks a start as + // not-yet-durably-settled whatever the snapshot currently says. If + // settlement ever moves ahead of the persisted summary, an operator + // sweeping a draining deployment reads a clean `resource-owners` while a + // start is still in flight, locks, and the deployment taking over resumes + // work that never stopped. This test fails the moment that ordering + // changes. + const sqlite = openSqlite(); + const binding = sqliteUnitDatabase(sqlite); + const storage = createD1Storage({ binding: binding as never }); + await createResourceOwnershipSchema(binding as never); + sqlite.exec( + `CREATE TABLE flowsafe_deployment ( + id INTEGER PRIMARY KEY CHECK (id = 1), + tenant_tag TEXT NOT NULL, + provisioned_at TEXT NOT NULL + )`, + ); + sqlite + .prepare( + 'INSERT INTO flowsafe_deployment (id, tenant_tag, provisioned_at) VALUES (1, ?, ?)', + ) + .run('acme', new Date(NOW).toISOString()); + + // A step that PARKS mid-execution, which is what makes the in-flight window + // observable at all: without it the start returns before any assertion can + // run, and the window this pins would never be open. + let announceStarted: () => void = () => undefined; + const started = new Promise((resolve) => { + announceStarted = resolve; + }); + let releaseStep: () => void = () => undefined; + const held = new Promise((resolve) => { + releaseStep = resolve; + }); + + const fence = new ExecutionFenceStore(binding as never); + await fence.seed('open'); + const reservations = new StartIdempotencyStore(binding as never); + const buildRuntime = (): RunnerRuntime => { + const { createWorkflow, createStep, runtime } = init( + { storage }, + { executionFence: fence, startIdempotency: reservations }, + ); + const gate = createStep({ + id: 'gate', + inputSchema: z.object({}), + outputSchema: z.object({}), + suspendSchema: z.object({ reason: z.string() }), + resumeSchema: z.object({ go: z.boolean() }), + execute: async ({ resumeData, suspend }) => { + if (resumeData) return {}; + announceStarted(); + await held; + return suspend({ reason: 'wait' }); + }, + }); + createWorkflow({ + id: 'gated', + inputSchema: z.object({}), + outputSchema: z.object({}), + }) + .then(gate) + .commit(); + return runtime; + }; + + interface OwnerEnv { + owners: D1ResourceOwnershipStore; + DEPLOYMENT_TENANT: string; + DEPLOYMENT_IDENTITY_SECRET: string; + DB: unknown; + } + class OwnerRunner extends DurableObjectRunner { + protected runOwnership(env: OwnerEnv): DurableObjectRunOwnershipStore { + return env.owners; + } + protected runLifecycle(): { abandonApprovals: () => Promise } { + return { abandonApprovals: async () => undefined }; + } + protected build(): RunnerRuntime { + return buildRuntime(); + } + } + const secret = 'inventory-ownership-pin-secret-00001'; + const runner = new OwnerRunner(undefined, { + owners: new D1ResourceOwnershipStore(binding as never), + DEPLOYMENT_TENANT: 'acme', + DEPLOYMENT_IDENTITY_SECRET: secret, + DB: binding, + }); + const runId = 'abc_inflight'; + const post = (path: string, body: unknown): Request => + new Request(`http://do${path}`, { + method: 'POST', + headers: { + 'content-type': 'application/json', + [DEPLOYMENT_IDENTITY_HEADER]: secret, + [EXECUTION_PRINCIPAL_HEADER]: JSON.stringify({ + kind: 'human', + id: 'ada', + role: 'operator', + }), + }, + body: JSON.stringify(body), + }); + const inventory = new DeploymentInventory(binding as InventoryDatabase, { + now: () => NOW, + }); + + // #when — the start is IN FLIGHT: ownership reserved, step executing, and + // nothing persisted yet. + const start = runner.fetch( + post('/runs', { workflowId: 'gated', runId, inputData: {} }), + ); + await started; + + // #then — the reservation is UNSETTLED while the start is in flight. This + // is the assertion the invariant lives in: it fails if settlement moves + // ahead of the persisted summary. + const inFlight = await inventory.read('resource-owners'); + expect(inFlight.entries).toEqual([ + { + key: ['run', runId], + detail: { owner_kind: 'human', owner_id: 'ada' }, + }, + ]); + expect(inFlight.count).toBe(1); + + // #then — and the executing run is not hidden from `runs` either: the + // engine's own `running` snapshot is already there. Recorded because the + // two categories overlap DURING execution and diverge only at settlement, + // which is what the next step asserts. + expect( + (await inventory.read('runs')).entries.map((entry) => [ + entry.key[1], + entry.detail.status, + ]), + ).toEqual([[runId, 'running']]); + + // #when — the step reaches its first suspend, so a summary persists and the + // reservation settles. + releaseStep(); + const summary = (await (await start).json()) as { status: string }; + expect(summary.status).toBe('suspended'); + + // #then — the categories DIVERGE at settlement: `runs` still carries the + // suspended run, and the settled reservation has left `resource-owners`. + expect( + (await inventory.read('runs')).entries.map((entry) => entry.key), + ).toEqual([['gated', runId]]); + expect((await inventory.read('resource-owners')).entries).toEqual([]); + + // #when — the run reaches a terminal state. + const resumed = await runner.fetch( + post(`/runs/gated/${runId}/resume`, { + step: 'gate', + resumeData: { go: true }, + requestedBy: 'reviewer-1', + requestedByKind: 'human', + }), + ); + expect(resumed.status).toBe(200); + + // #then — both categories are empty, which is what a drain proof reads as + // permission to lock. + expect((await inventory.read('runs')).entries).toEqual([]); + expect((await inventory.read('resource-owners')).entries).toEqual([]); + }); +}); diff --git a/packages/flowsafe/src/do-runner/inventory.ts b/packages/flowsafe/src/do-runner/inventory.ts new file mode 100644 index 0000000..432b7a7 --- /dev/null +++ b/packages/flowsafe/src/do-runner/inventory.ts @@ -0,0 +1,1182 @@ +// SPDX-License-Identifier: Apache-2.0 +// The deployment drain inventory — the read an operator holds a fence open on +// while they prove this deployment is safe to leave behind. +// +// WHY it exists. The fence (execution-fence.ts) can stop a deployment minting +// work, but stopping is not finishing. A migration only becomes safe at the +// moment nobody can point at a run, an approval, a queued task, or a due +// dispatch that this deployment still owes. Before this module the only way to +// answer that was to name the tables you happened to remember and count them by +// hand — an answer whose failure mode is silence: a table nobody thought of +// holds a suspended run, the migration proceeds, and two deployments resume it. +// +// THE TAXONOMY IS THE CONTRACT, and completeness is the product. What is sold +// here is not a set of queries; it is the claim that these queries are ALL of +// them. That claim cannot be maintained by care, so it is maintained by the +// census: `FLOWSAFE_TABLES` below maps every flowsafe-owned table to either an +// inventory category or a written reason it holds no work, and the census test +// (do-runner/mastra-schema-guard.test.ts) fails CI when a table — ours or a new +// one arriving with a @mastra/core bump — appears in neither list. A future +// author adding a work-holding table therefore cannot ship it silently: they +// must either inventory it or state, in more than twenty characters, why an +// operator draining a deployment may ignore it. The guarantee an operator reads +// off an empty inventory is only ever as good as that obligation. +// +// TWO CLASSES, because a drain proof defined over everything can never pass. +// `work` is what must reach empty: runs, approvals awaiting a decision, queued +// tasks, due dispatches, unsettled reservations. `standing` is configuration +// that ARMS future work — schedules, provider subscriptions — and by design it +// never empties; a deployment with three cron schedules is drainable, and +// demanding otherwise would make the proof unreachable rather than strict. +// +// ZERO SIDE EFFECTS, MECHANICALLY. Every read here is a bare SELECT. Nothing +// calls `#ensureSchema`, because a lazy `CREATE TABLE IF NOT EXISTS` is a WRITE +// and the whole value of this surface is that an operator can run it against a +// deployment they are about to copy without changing what they are copying. A +// missing table is not a fault but an empty category (`missingTableReadsEmpty`, +// shared with the fence and the reservation store) — every table here is +// created lazily by the first feature that uses it, so "absent" and "holds +// nothing" are the same sentence. The database seam this module accepts +// (`InventoryDatabase`) deliberately has no `run()`: the read-only property is +// in the TYPE, not only in the review. +// +// WHY NOT REUSE THE EXISTING READERS. Every domain already has a list method — +// `statusFor`, the run-router's GET, `manager.listTasks`, the schedule store's +// `listTriggers`. None of them is usable here, and not for style reasons: each +// either initializes schema on the way in, or advances a cursor, or reaches a +// Durable Object that would wake and re-arm an alarm. An inventory built on +// them would perturb the very deployment it is measuring. So the queries below +// are new and pure, and each one's predicate is derived from the production +// reader or purge that owns the same notion — the terminal-run SQL is literally +// the retention purge's fragment, the due-notification predicate is +// `listDueNotifications`'s, the deferred-dispatch predicate is the schedule +// store's own dispatch guard — so "finished" cannot come to mean one thing to +// the code that acts and another to the code that certifies. + +import { + DEPLOYMENT_SENTINEL_TABLE, + EXECUTION_FENCE_TABLE, +} from '#deployment-identity-protocol'; +import { APPROVALS_TABLE, OPEN_STATUSES } from '../approval-api/types.js'; +import { missingTableReadsEmpty } from './cause-chain.js'; +import { + BACKGROUND_TASK_TERMINAL_STATUSES, + RESOURCE_OWNER_TABLE, + RUN_TERMINAL_SNAPSHOT_SQL, + RUN_TERMINAL_STATUSES, +} from './d1-storage.js'; +import { DoStatusError } from './do-status-error.js'; +import { + EXECUTION_FENCE_SUSPEND_KEY, + type ExecutionFenceState, +} from './execution-fence.js'; +import { DUE_NOTIFICATION_SQL } from './notification-predicate.js'; +import { + START_IDEMPOTENCY_TABLE, + START_RESERVATION_STATES, +} from './start-idempotency.js'; +import { validateTablePrefix } from './table-prefix.js'; + +/** + * The Mastra-owned tables this module reads, spelled here because they belong + * to a layer this file may not import from. + * + * `mastra_schedules` and `mastra_schedule_triggers` are created by + * `schedules/schedules-d1.ts`, and `mastra_notifications` by + * `signals/notifications-d1.ts` — both of which are built ON do-runner, so + * importing their storage modules would close a cycle and pull their runtime + * dependencies into the public `./do-runner` graph. The pending-notification + * predicate is therefore single-homed in the import-free + * `do-runner/notification-predicate.ts` leaf, which both readers can safely + * import. The census test can see both sides at once, and crosses every name + * below against the storage inventory the schema guard pins, so a rename fails + * there rather than turning a category permanently empty. + * `mastra_workflow_snapshot` and `mastra_background_tasks` come from the + * @mastra/cloudflare-d1 adapter and are pinned the same way. + */ +const SNAPSHOT_TABLE = 'mastra_workflow_snapshot'; +const BACKGROUND_TASKS_TABLE = 'mastra_background_tasks'; +const NOTIFICATIONS_TABLE = 'mastra_notifications'; +const SCHEDULES_TABLE = 'mastra_schedules'; +const SCHEDULE_TRIGGERS_TABLE = 'mastra_schedule_triggers'; + +/** + * The subscription registry's table. Same reason as the block above: + * `signal-providers/subscription-d1.ts` imports do-runner, so the name travels + * to the census rather than to this file. + */ +const SIGNAL_SUBSCRIPTIONS_INVENTORY_TABLE = 'flowsafe_signal_subscriptions'; + +/** + * Whether a category must reach empty before a deployment may be locked. + * + * `work` is outstanding obligation: something is expected to happen and has not + * happened yet. `standing` is configuration that survives the migration and is + * reported so an operator can reconcile it on the far side, never so they can + * wait for it to disappear. + */ +export type InventoryCategoryClass = 'work' | 'standing'; + +/** + * Every category name, in the order the index reports them: work first, then + * standing. + */ +export const INVENTORY_CATEGORIES = [ + 'runs', + 'approvals-waiting', + 'schedule-deferred-dispatches', + 'pending-notifications', + 'background-tasks', + 'resource-owners', + 'start-reservations', + 'schedules', + 'signal-subscriptions', +] as const; + +/** One inventory category's name. */ +export type InventoryCategory = (typeof INVENTORY_CATEGORIES)[number]; + +/** What the index says about one category, before anything is counted. */ +export interface InventoryCategoryDescriptor { + /** The category to pass as `?category=`. */ + readonly category: InventoryCategory; + /** Whether a drain must see this category empty. */ + readonly class: InventoryCategoryClass; + /** The table its rows live in, un-prefixed. */ + readonly table: string; + /** What a row in this category means, in one sentence. */ + readonly holds: string; +} + +/** One row, projected to what an operator needs to act on it. */ +export interface InventoryEntry { + /** + * The row's primary-key components, in the order the keyset scan visits + * them. Passing the LAST entry's key back as `cursor` resumes after it. + */ + readonly key: readonly string[]; + /** Category-specific columns; absent values are omitted rather than nulled. */ + readonly detail: Readonly>; +} + +/** One page of one category. */ +export interface InventoryPage { + readonly category: InventoryCategory; + readonly class: InventoryCategoryClass; + /** The table read, with the deployment's storage prefix applied. */ + readonly table: string; + readonly entries: readonly InventoryEntry[]; + /** + * Pass back as `?cursor=` to continue. Absent means this page reached the + * end of the category as of this read — which, for a `work` category, is the + * observation a drain proof is built from. + */ + readonly cursor?: string; + /** + * Total rows matching this category, counted on the FIRST page of a sweep + * only (a caller that is paging has already committed to walking them). + * Absent on continuation pages, and absent when the table does not exist. + */ + readonly count?: number; + /** + * Category-specific totals taken with `count`, in the same read: the numbers + * that answer an operator's next question without a second sweep. See + * `background-tasks` (`fenceSuspended`) and `pending-notifications` + * (`notDue`). + */ + readonly totals?: Readonly>; +} + +/** + * State that holds work but cannot be counted, declared rather than omitted. + * + * An inventory that silently skipped these would be reporting a smaller number + * than the truth under the same name as a complete one, which is the single + * most dangerous thing this surface could do. Each entry states what it holds + * and the BOUND on how long it can stay invisible, so the drain procedure can + * be built to outlast it. + */ +export interface UnenumerableState { + /** Stable identifier for the invisible state. */ + readonly name: string; + /** What it can hold. */ + readonly holds: string; + /** Why no pure query can see it. */ + readonly because: string; + /** How long it can stay invisible, and what makes it visible again. */ + readonly bound: string; +} + +/** The rule an operator reads an empty inventory under. */ +export interface DrainProofContract { + /** + * How to interpret point-in-time readings while draining still admits work. + * Rows can enter or leave, but an empty result cannot over-count work. + */ + readonly reading: string; + /** What a proof actually is. */ + readonly proof: string; + /** The fence states an empty `work` set is reachable from. */ + readonly reachableFrom: readonly ExecutionFenceState[]; + /** How to interpret a post-lock re-sweep and a read taken under the lock. */ + readonly note: string; +} + +/** The no-category response: what can be asked for, and how to read the answer. */ +export interface InventoryIndex { + readonly categories: readonly InventoryCategoryDescriptor[]; + readonly unenumerable: readonly UnenumerableState[]; + readonly drainProof: DrainProofContract; +} + +/** + * The database surface an inventory read needs — and NOTHING ELSE. + * + * There is no `run()` on this seam, deliberately. Every other store in this + * package takes a shape that can write, because it writes; this one cannot + * express a write at all, so "the inventory performs no writes" is a fact about + * the type rather than a promise about the code. `env.DB` satisfies it + * structurally, as does a node:sqlite test double. + */ +export interface InventoryDatabase { + prepare(query: string): InventoryStatement; +} + +/** One prepared read. */ +export interface InventoryStatement { + bind(...values: unknown[]): InventoryStatement; + all(): Promise<{ results: T[] }>; +} + +/** How one flowsafe-owned table is accounted for by the inventory. */ +export type InventoryTableAccounting = + | { + /** Rows of this table are reported under this category. */ + readonly category: InventoryCategory; + } + | { + /** + * This table holds no drainable work, and here is why. The census + * requires more than twenty characters: "not work" and "nobody looked" + * are indistinguishable at that length, and only one of them is a + * decision. + */ + readonly excluded: string; + }; + +/** One flowsafe-owned table and its accounting. */ +export interface FlowsafeTableEntry { + /** The un-prefixed table name. */ + readonly table: string; + /** What the table is for. */ + readonly purpose: string; + /** Its inventory category, or the reason it has none. */ + readonly accounting: InventoryTableAccounting; +} + +/** + * THE FLOWSAFE TABLE CENSUS. Every `flowsafe_`-prefixed table this package + * creates, mapped to an inventory category or to a justified exclusion. + * + * It exists because the schema guard's `mastra_%` sweep cannot see these: they + * are not created by `createD1Storage`, and two of them (the deployment + * sentinel and the fence) are created by the provisioning protocol before any + * runtime code runs. Without an explicit list, a flowsafe table holding + * suspended work could be added, purged, documented, and shipped without ever + * meeting the question this module exists to answer. + * + * ADDING A TABLE: add it here in the same change. The census test fails until + * you do, and it fails again if you claim an exclusion in fewer than twenty + * characters. + */ +export const FLOWSAFE_TABLES: readonly FlowsafeTableEntry[] = [ + { + table: APPROVALS_TABLE, + purpose: 'the durable approval queue — one record per suspended decision', + accounting: { category: 'approvals-waiting' }, + }, + { + table: DEPLOYMENT_SENTINEL_TABLE, + purpose: + 'the ownership sentinel naming the tenant this database belongs to', + accounting: { + excluded: + 'deployment identity, not work: one row written once at provisioning, read to prove the bindings match the database. It is COPIED by a migration, never drained.', + }, + }, + { + table: EXECUTION_FENCE_TABLE, + purpose: 'the deployment execution fence — one row holding its state', + accounting: { + excluded: + 'the control this inventory is read under. Its single row is the instrument, not a measurement: counting it would make every drain proof fail on the fence that is holding the drain open.', + }, + }, + { + table: 'flowsafe_notification_sequence', + purpose: 'the monotonic insertion ordinal the notification inbox orders by', + accounting: { + excluded: + 'a single counter row, not a unit of work. The notifications it orders are inventoried under pending-notifications; the counter itself has nothing to finish and is re-derivable from the inbox.', + }, + }, + { + table: RESOURCE_OWNER_TABLE, + purpose: + 'the resource-ownership registry — who holds a run, thread, resource, or schedule', + accounting: { category: 'resource-owners' }, + }, + { + table: SIGNAL_SUBSCRIPTIONS_INVENTORY_TABLE, + purpose: 'armed provider subscriptions routing external signals to threads', + accounting: { category: 'signal-subscriptions' }, + }, + { + table: START_IDEMPOTENCY_TABLE, + purpose: 'owner-bound start reservations — one row per idempotency key', + accounting: { category: 'start-reservations' }, + }, +]; + +/** The largest page an inventory read will return, whatever `limit` asks for. */ +export const INVENTORY_MAX_LIMIT = 200; + +/** The page size used when `limit` is absent. */ +export const INVENTORY_DEFAULT_LIMIT = 100; + +/** Longest `cursor` string accepted, before it is parsed. */ +const MAX_CURSOR_LENGTH = 1_024; + +/** + * Bound parameters D1 accepts in ONE statement. The retention purge caps its + * own batch against the same limit (d1-storage.ts). + */ +export const D1_MAX_BOUND_PARAMETERS = 100; + +/** + * Bound parameters the run-owner lookup spends on anything OTHER than run ids. + * + * Zero today: `resource_kind = 'run'` is a SQL literal, not a bind. It is named + * rather than assumed because the slice size below is `cap - this`, and a + * future predicate that binds a value would otherwise push the statement one + * parameter over the limit with nothing to catch it. + */ +export const RUN_OWNER_FIXED_BINDINGS = 0; + +/** + * Run ids per owner-lookup statement: the full parameter budget, less whatever + * the statement spends on its own predicate. 100 - 0 = 100, so a maximum page + * (INVENTORY_MAX_LIMIT = 200) costs exactly two statements. + */ +export const RUN_OWNER_LOOKUP_CHUNK = + D1_MAX_BOUND_PARAMETERS - RUN_OWNER_FIXED_BINDINGS; + +/** + * A malformed inventory request: an unknown category, an unparseable cursor, or + * a cursor whose shape does not match the category it was sent to. + * + * 400 rather than a silent restart from the beginning. A cursor that quietly + * resets would make a sweep re-read rows it had already counted and never + * reach the end, and an operator watching for two consecutive empty sweeps + * would wait forever without being told why. + */ +export class InvalidInventoryRequestError extends DoStatusError { + readonly status = 400; + readonly reason: { readonly code: 'INVALID_INVENTORY_REQUEST' }; + + constructor(message: string) { + super(message); + this.name = 'InvalidInventoryRequestError'; + this.reason = { code: 'INVALID_INVENTORY_REQUEST' }; + } +} + +/** Is this a category the inventory serves? */ +export function isInventoryCategory( + value: unknown, +): value is InventoryCategory { + return (INVENTORY_CATEGORIES as readonly string[]).includes(value as string); +} + +/** + * The state a pure D1 query cannot see, declared rather than omitted. + * + * THE SWEEP BEHIND THIS LIST. Every key a Durable Object in this package writes + * was checked against the categories above, and all but one is backed by a row + * a query here already returns: + * + * `flowsafe:run-owner-recovery:v1` — DECLARED below. It is written BEFORE the + * D1 owner row, so a crash between the two leaves a run with no D1 record at + * all. + * + * `flowsafe:agent-thread-binding:v1` / `flowsafe:agent-run:v1:*` — a thread's + * binding to its agent run and that run's own state. Both name a run whose + * ownership is RESERVED in D1 BEFORE execution begins + * (agent-host/thread-host.ts, durable-object.ts) and settled only AFTER a + * summary has persisted. The engine also writes a `running` snapshot as the + * run starts, so an executing run is normally reported under `runs` on its own + * — but that snapshot lands INSIDE `runtime.start`, after the reservation, and + * the unsettled reservation is what covers the gap between the two as well as + * any later moment the snapshot is behind the run. `resource-owners` is a WORK + * category for exactly that reason: an unsettled reservation means a start + * that has not durably settled, whatever the snapshot currently says. + * (Pinned behaviourally by inventory.test.ts's ownership-ordering test, which + * fails if settlement ever moves ahead of the persisted summary.) + * + * `flowsafe:suspension-deadline:v1` — an armed wake for a run that is + * suspended, and therefore nonterminal, and therefore under `runs`. + * + * `flowsafe:maintenance-health:v1`, `flowsafe:maintenance-nonces:v1`, + * `flowsafe:maintenance-deadline-cursor:v1` — the maintenance object's own + * health, replay protection, and scan position. None is a unit of work: they + * describe the sweeper, not what it sweeps. + */ +export const INVENTORY_UNENUMERABLE: readonly UnenumerableState[] = [ + { + name: 'run-owner-recovery-journal', + holds: + 'a run whose Durable Object journalled its recovery key but had not yet written its D1 owner row', + because: + 'the journal lives in the run object own storage; D1 has no row for it yet, so no query over this database can observe it', + bound: + '60,000 ms (60 seconds), the RUN_OWNER_RECOVERY_DELAY_MS cadence in do-runner/durable-object.ts: the object reconciles the journal on its next wake, after which the run appears under `runs` (or has finished). Two sweeps at least that far apart therefore cover the window.', + }, + { + name: 'persisted-idle-signals', + holds: + 'signals a draining deployment persisted instead of waking a run for', + because: + 'a persisted signal is written into agent MEMORY as a message, carrying no consumption marker any predicate could test — it is indistinguishable, in storage, from ordinary conversation history', + bound: + 'none is needed: these are deliberately NOT drainable. The fence degrades an idle wake to a persist precisely so the signal survives the migration and is picked up by the deployment that takes over, so an operator must not wait for them to disappear.', + }, +]; + +/** + * The rule that turns inventory readings into a decision. + * + * Stated in the index response, and repeated in the route doc, because every + * dangerous use of this surface is a use that skips it: one empty sweep is not + * a proof, and an empty sweep taken under `migration-locked` is not even a + * measurement of a drain. + */ +export const INVENTORY_DRAIN_PROOF: DrainProofContract = { + reading: + 'Every reading is a point-in-time observation, not a snapshot: rows can enter or leave work categories while draining admits work. Empty results cannot over-count, and keyset pagination never skips a row that existed before the sweep started.', + proof: + 'The proof is TWO consecutive full sweeps, at least 60,000 ms (60 seconds) apart — the RUN_OWNER_RECOVERY_DELAY_MS cadence in do-runner/durable-object.ts — in which every `work` category returns no entries. One sweep cannot cover the Durable Object journal window; two, spaced that far, can.', + reachableFrom: ['draining'], + note: "The two-sweep proof is taken from 'draining', where work can finish. A host that needs a hard guarantee can re-sweep once after transitioning to 'migration-locked': an empty post-lock sweep is conclusive; a non-empty one means work is still outstanding, either because it entered after the proof or because the lock parked it before it finished. Return to draining and repeat the proof. An inventory read taken under 'migration-locked' measures what the fence parked rather than what the deployment would otherwise be doing.", +}; + +/** Every category's descriptor, in index order. */ +export const INVENTORY_CATEGORY_DESCRIPTORS: readonly InventoryCategoryDescriptor[] = + [ + { + category: 'runs', + class: 'work', + table: SNAPSHOT_TABLE, + holds: + 'a workflow or agent run that has not reached a terminal state (cancelled and timed_out count as live until their lifecycle cleanup completes)', + }, + { + category: 'approvals-waiting', + class: 'work', + table: APPROVALS_TABLE, + holds: + 'an approval still awaiting a decision — pending, claimed, or escalated', + }, + { + category: 'schedule-deferred-dispatches', + class: 'work', + table: SCHEDULE_TRIGGERS_TABLE, + holds: + 'a schedule fire that was claimed and deferred but never dispatched; settled fire history in the same table is excluded', + }, + { + category: 'pending-notifications', + class: 'work', + table: NOTIFICATIONS_TABLE, + holds: + 'an agent-inbox notification that is pending and already due for dispatch', + }, + { + category: 'background-tasks', + class: 'work', + table: BACKGROUND_TASKS_TABLE, + holds: + 'a background task that has not settled — queued, running, or suspended (including parked by the fence)', + }, + { + category: 'resource-owners', + class: 'work', + table: RESOURCE_OWNER_TABLE, + holds: + 'an ownership reservation that was taken but never settled or released', + }, + { + category: 'start-reservations', + class: 'work', + table: START_IDEMPOTENCY_TABLE, + holds: + 'an idempotency key that has been reserved or claimed but whose run has not settled', + }, + { + category: 'schedules', + class: 'standing', + table: SCHEDULES_TABLE, + holds: + 'a schedule that will arm future work — standing configuration a migration carries, never drains', + }, + { + category: 'signal-subscriptions', + class: 'standing', + table: SIGNAL_SUBSCRIPTIONS_INVENTORY_TABLE, + holds: + 'an armed provider subscription routing an external resource to a thread — carried, not drained', + }, + ]; + +/** The whole no-category answer, assembled once. */ +export const INVENTORY_INDEX: InventoryIndex = { + categories: INVENTORY_CATEGORY_DESCRIPTORS, + unenumerable: INVENTORY_UNENUMERABLE, + drainProof: INVENTORY_DRAIN_PROOF, +}; + +/** Construction-time options for a deployment inventory. */ +export interface DeploymentInventoryOptions { + /** + * The same storage table prefix `createD1Storage` and the retention purges + * take. It applies to the Mastra-owned tables only, exactly as it does + * there — the flowsafe-owned registries are never prefixed. + */ + tablePrefix?: string; + /** Injectable clock, for the due-notification predicate. */ + now?: () => number; +} + +/** Per-read options: where to resume, and how many rows to take. */ +export interface InventoryReadOptions { + /** The `key` of the last entry of the previous page, as JSON text. */ + cursor?: string; + /** Rows to return, clamped to INVENTORY_MAX_LIMIT. */ + limit?: number; +} + +/** How one category is scanned, before any row is read. */ +interface CategoryQuery { + /** Primary-key columns, in the order the keyset visits them. */ + readonly key: readonly string[]; + /** `expression AS alias` projections carried into `detail`. */ + readonly detail: readonly string[]; + /** Bindings the `detail` projections need, in order. */ + readonly detailBinds?: readonly unknown[]; + /** + * Aliases from `detail` that are 1/0 flags, reported as booleans. SQLite has + * no boolean type, so the projection cannot say so itself. + */ + readonly flags?: readonly string[]; + /** The category's predicate, with `?` placeholders. */ + readonly where: string; + /** Bindings for `where`, in order. */ + readonly binds: readonly unknown[]; + /** + * Extra `SUM(...) AS alias` aggregates taken with the count on the first + * page. Their predicate is the category's own, so a total here is always a + * partition of `count` unless its doc says otherwise. + */ + readonly totals?: readonly string[]; + /** Bindings the `totals` expressions need, in order. */ + readonly totalsBinds?: readonly unknown[]; + /** + * A WIDER predicate for the aggregate pass, when a category's totals must + * describe rows the page deliberately excludes — `pending-notifications` + * pages only what is DUE, but "how many are pending and not due yet" is the + * question an operator asks next and no amount of paging answers it. + * + * Widening the aggregate does NOT widen `count`: a category that sets this + * must also set `countExpression`, so `count` still means "rows in this + * category" everywhere. A count that meant one thing for eight categories + * and something larger for the ninth is a number an operator would read + * wrong exactly once. + */ + readonly countWhere?: string; + /** + * How `count` is computed when `countWhere` is wider than `where` — a + * conditional SUM that re-applies the category's own predicate. + */ + readonly countExpression?: string; + /** Bindings for `countExpression`, in order. */ + readonly countExpressionBinds?: readonly unknown[]; + /** Bindings for `countWhere`, when it differs. */ + readonly countBinds?: readonly unknown[]; + /** + * A second, strictly optional read that decorates the page. Used for the run + * owner annotation, whose table is a different feature's and may not exist — + * so it is separate from the page query rather than joined into it: a + * deployment that never wired ownership must still be able to prove it holds + * no runs. + */ + readonly annotate?: ( + db: InventoryDatabase, + entries: readonly MutableInventoryEntry[], + ) => Promise; +} + +/** + * An entry while it is still being built. Identical to `InventoryEntry` except + * that `detail` is writable, which is what lets an annotation add a column the + * page query could not safely join. + */ +interface MutableInventoryEntry { + readonly key: readonly string[]; + readonly detail: Record; +} + +type RawRow = Record; + +function placeholdersFor(values: readonly unknown[]): string { + return values.map(() => '?').join(', '); +} + +/** + * A keyset predicate over an arbitrary number of key columns, written as + * nested comparisons rather than SQLite's row-value syntax. + * + * Row values (`(a, b) > (?, ?)`) would be shorter and SQLite has supported them + * since 3.15, but the expansion below is what every index in this package is + * already shaped for and what any SQLite the adapter might sit on understands. + * The shape is strictly-greater at every level, so a cursor never re-reads its + * own row and never skips the one after it. + */ +function keysetPredicate(columns: readonly string[]): string { + const clauses: string[] = []; + for (let index = 0; index < columns.length; index += 1) { + const equalities = columns + .slice(0, index) + .map((column) => `${column} = ?`) + .join(' AND '); + const greater = `${columns[index]} > ?`; + clauses.push( + equalities === '' ? greater : `(${equalities} AND ${greater})`, + ); + } + return `(${clauses.join(' OR ')})`; +} + +/** The bindings `keysetPredicate` expects, for one cursor. */ +function keysetBindings(cursor: readonly string[]): unknown[] { + const binds: unknown[] = []; + for (let index = 0; index < cursor.length; index += 1) { + for (let prefix = 0; prefix < index; prefix += 1) { + binds.push(cursor[prefix]); + } + binds.push(cursor[index]); + } + return binds; +} + +/** + * Parse a caller's cursor into the key it names, refusing anything that is not + * exactly one key for THIS category. + * + * Refusing rather than ignoring: a cursor from a different category, or one + * that lost a component, would resume the scan somewhere the caller did not + * ask for — and a sweep that silently restarts is a sweep that never proves + * emptiness. + */ +function parseCursor( + cursor: string | undefined, + category: InventoryCategory, + arity: number, +): readonly string[] | undefined { + if (cursor === undefined || cursor === '') return undefined; + if (cursor.length > MAX_CURSOR_LENGTH) { + throw new InvalidInventoryRequestError('inventory cursor exceeds limit'); + } + let parsed: unknown; + try { + parsed = JSON.parse(cursor); + } catch { + throw new InvalidInventoryRequestError( + `inventory cursor for '${category}' is not valid JSON`, + ); + } + if ( + !Array.isArray(parsed) || + parsed.length !== arity || + !parsed.every((part) => typeof part === 'string') + ) { + throw new InvalidInventoryRequestError( + `inventory cursor for '${category}' must name ${String(arity)} key component(s)`, + ); + } + return parsed as readonly string[]; +} + +function clampLimit(limit: number | undefined): number { + if (limit === undefined) return INVENTORY_DEFAULT_LIMIT; + if (!Number.isFinite(limit) || !Number.isInteger(limit) || limit < 1) { + throw new InvalidInventoryRequestError( + 'inventory limit must be a positive integer', + ); + } + return Math.min(limit, INVENTORY_MAX_LIMIT); +} + +/** + * Project one row's non-key columns, dropping anything absent. + * + * NULL columns are omitted rather than reported as `null`, the same rule the + * fence's reading payload follows: a caller reading `runId: null` off a + * deferred dispatch would have to know that null means "not dispatched" rather + * than "column missing", and the absent field says it without ambiguity. + */ +function detailOf( + row: RawRow, + keys: readonly string[], + flags: readonly string[], +): Record { + const detail: Record = {}; + for (const [column, value] of Object.entries(row)) { + if (keys.includes(column)) continue; + if (value === null || value === undefined) continue; + if (flags.includes(column)) { + detail[column] = value !== 0; + continue; + } + if ( + typeof value === 'string' || + typeof value === 'number' || + typeof value === 'boolean' + ) { + detail[column] = value; + } + } + return detail; +} + +function keyOf(row: RawRow, keys: readonly string[]): readonly string[] { + return keys.map((column) => String(row[column] ?? '')); +} + +/** + * 1 when a background task's suspend payload carries the fence marker. + * + * Written once and used in both the projection and the aggregate so a row + * flagged on the page and a row counted in the total can never be decided by + * two different expressions. `json_valid` guards a NULL or malformed payload, + * which json_extract would otherwise throw on — a tool free to suspend with any + * payload it likes is a tool that could otherwise wedge this sweep. + */ +function fenceParkedSql(): string { + return `CASE WHEN json_valid(suspend_payload) + AND json_extract(suspend_payload, '$."${EXECUTION_FENCE_SUSPEND_KEY}"') IS NOT NULL + THEN 1 ELSE 0 END`; +} + +/** + * Decorate a page of runs with the principal that owns each one. + * + * A SECOND query rather than a join, for two reasons that both point the same + * way. The ownership registry belongs to the approval layer and a deployment + * that never wired it has no such table — a join would turn its absence into a + * failure of the `runs` category, which is the one category a drain proof + * cannot do without. And the annotation is decoration: an operator deciding + * whether to lock needs the run, and wants the owner. So a missing table, and + * only a missing table, degrades to unannotated rows rather than to no rows. + * + * Ownership is keyed by run id alone (`resource_kind = 'run'`), so the lookup + * takes the page's run ids — never an unbounded scan. + * + * CHUNKED, because a page can be larger than one statement may bind. A caller + * asking for `limit=200` (INVENTORY_MAX_LIMIT, a real knob) would otherwise + * build a 200-parameter `IN` list, which real D1 refuses outright while + * node:sqlite — with its 32766-variable ceiling — accepts happily. That + * combination is the worst one available: the suite stays green and the route + * answers a generic 500 on the deployment an operator is trying to drain. + */ +async function annotateRunOwners( + db: InventoryDatabase, + entries: readonly MutableInventoryEntry[], +): Promise { + if (entries.length === 0) return; + // key = [workflow_name, run_id]; ownership is keyed on the run id. + const runIds = entries.map((entry) => entry.key[1] ?? ''); + const owners = new Map(); + for ( + let offset = 0; + offset < runIds.length; + offset += RUN_OWNER_LOOKUP_CHUNK + ) { + const slice = runIds.slice(offset, offset + RUN_OWNER_LOOKUP_CHUNK); + let rows: RawRow[]; + try { + ({ results: rows } = await db + .prepare( + `SELECT resource_id, owner_kind, owner_id + FROM ${RESOURCE_OWNER_TABLE} + WHERE resource_kind = 'run' + AND resource_id IN (${placeholdersFor(slice)})`, + ) + .bind(...slice) + .all()); + } catch (error) { + // A deployment with no ownership registry has none for ANY slice, so the + // whole annotation is abandoned rather than the current chunk: continuing + // would re-throw the same absence once per hundred rows. + if (missingTableReadsEmpty(error, RESOURCE_OWNER_TABLE)) return; + throw error; + } + for (const row of rows) { + const kind = row.owner_kind; + const id = row.owner_id; + if (typeof kind !== 'string' || typeof id !== 'string') continue; + owners.set(String(row.resource_id), `${kind}:${id}`); + } + } + for (const entry of entries) { + const owner = owners.get(entry.key[1] ?? ''); + if (owner !== undefined) entry.detail.owner = owner; + } +} + +/** + * The deployment's read-only work inventory over one D1 database — the same + * database its runs, approvals, and fence live in, so what it reports and what + * the fence governs cannot be two different deployments. + */ +export class DeploymentInventory { + readonly #db: InventoryDatabase; + readonly #prefix: string; + readonly #now: () => number; + + constructor(db: InventoryDatabase, options: DeploymentInventoryOptions = {}) { + this.#db = db; + this.#prefix = + validateTablePrefix(options.tablePrefix, 'tablePrefix') ?? ''; + this.#now = options.now ?? Date.now; + } + + /** + * What this deployment can be asked about, what it cannot answer, and the + * rule for reading an empty answer. + * + * A constant, and deliberately not a query: the index is the CONTRACT, and a + * contract that changed with the data would let an operator conclude that a + * category they cannot see does not apply to them. + */ + index(): InventoryIndex { + return INVENTORY_INDEX; + } + + /** One page of one category. */ + async read( + category: InventoryCategory, + options: InventoryReadOptions = {}, + ): Promise { + const descriptor = INVENTORY_CATEGORY_DESCRIPTORS.find( + (entry) => entry.category === category, + ); + if (!descriptor) { + throw new InvalidInventoryRequestError( + `unknown inventory category '${String(category)}'`, + ); + } + const query = this.#queryFor(category); + const table = this.#tableFor(descriptor); + const cursor = parseCursor(options.cursor, category, query.key.length); + const limit = clampLimit(options.limit); + const base: Pick = { + category, + class: descriptor.class, + table, + }; + + let rows: RawRow[]; + try { + rows = await this.#page(table, query, cursor, limit); + } catch (error) { + // A table nothing has created yet holds nothing. Every table here is made + // lazily by the first feature that writes it, so the absence is the same + // observation as an empty scan — and answering it with a fault would make + // a fresh deployment undrainable. + if (missingTableReadsEmpty(error, table)) return { ...base, entries: [] }; + throw error; + } + const entries: MutableInventoryEntry[] = rows.map((row) => ({ + key: keyOf(row, query.key), + detail: detailOf(row, query.key, query.flags ?? []), + })); + await query.annotate?.(this.#db, entries); + // A cursor is offered only when the page filled: a short page reached the + // end of the category as of this read, and that is the observation the + // drain proof is built on. + const next = + entries.length === limit ? entries[entries.length - 1] : undefined; + const counted = + cursor === undefined ? await this.#counts(table, query) : undefined; + return { + ...base, + entries, + ...(next === undefined ? {} : { cursor: JSON.stringify(next.key) }), + ...(counted === undefined ? {} : counted), + }; + } + + /** Every category's first page, for a caller sweeping the whole deployment. */ + async sweep( + options: { limit?: number } = {}, + ): Promise { + const pages: InventoryPage[] = []; + for (const descriptor of INVENTORY_CATEGORY_DESCRIPTORS) { + pages.push( + await this.read(descriptor.category, { + ...(options.limit === undefined ? {} : { limit: options.limit }), + }), + ); + } + return pages; + } + + #tableFor(descriptor: InventoryCategoryDescriptor): string { + // The prefix belongs to the Mastra-owned storage domains only — the same + // split createD1Storage and the retention purges take. A flowsafe registry + // is never prefixed, so prefixing it here would read an empty category on + // every prefixed deployment. + return descriptor.table.startsWith('mastra_') + ? `${this.#prefix}${descriptor.table}` + : descriptor.table; + } + + async #page( + table: string, + query: CategoryQuery, + cursor: readonly string[] | undefined, + limit: number, + ): Promise { + const projection = [...query.key, ...query.detail].join(', '); + const keyset = + cursor === undefined ? '' : ` AND ${keysetPredicate(query.key)}`; + const { results } = await this.#db + .prepare( + `SELECT ${projection} + FROM ${table} + WHERE ${query.where}${keyset} + ORDER BY ${query.key.join(', ')} + LIMIT ?`, + ) + .bind( + ...(query.detailBinds ?? []), + ...query.binds, + ...(cursor === undefined ? [] : keysetBindings(cursor)), + limit, + ) + .all(); + return results; + } + + async #counts( + table: string, + query: CategoryQuery, + ): Promise | undefined> { + const extra = query.totals ?? []; + // `count` is always "rows in this category", even when the aggregate scans + // wider: a category that widens `countWhere` re-applies its own predicate + // through `countExpression`. + const total = query.countExpression ?? 'COUNT(*)'; + const { results } = await this.#db + .prepare( + `SELECT ${total} AS total${extra.length === 0 ? '' : `, ${extra.join(', ')}`} + FROM ${table} + WHERE ${query.countWhere ?? query.where}`, + ) + .bind( + ...(query.countExpressionBinds ?? []), + ...(query.totalsBinds ?? []), + ...(query.countBinds ?? query.binds), + ) + .all(); + const row = results[0]; + if (row === undefined) return undefined; + const totals: Record = {}; + for (const [column, value] of Object.entries(row)) { + if (column === 'total') continue; + totals[column] = typeof value === 'number' ? value : 0; + } + return { + count: typeof row.total === 'number' ? row.total : 0, + ...(Object.keys(totals).length === 0 ? {} : { totals }), + }; + } + + #queryFor(category: InventoryCategory): CategoryQuery { + switch (category) { + case 'runs': + return { + key: ['workflow_name', 'run_id'], + detail: [ + // The same json_valid guard the retention purge uses, for the same + // reason: json_extract THROWS on malformed JSON, and one corrupt + // snapshot must not abort the sweep that is trying to account for + // the deployment. + `CASE WHEN json_valid(snapshot) + THEN json_extract(snapshot, '$.status') END AS status`, + 'updatedAt', + ], + // The predicate is the NEGATION of the retention purge's own terminal + // fragment, so "finished" means one thing on this deployment. An + // unclassifiable snapshot (invalid JSON, or valid JSON with no + // status) counts as WORK: a row nobody can prove is finished is + // exactly the row a migration must not walk away from. + where: `CASE WHEN json_valid(snapshot) + THEN CASE WHEN (${RUN_TERMINAL_SNAPSHOT_SQL}) THEN 0 ELSE 1 END + ELSE 1 END`, + binds: RUN_TERMINAL_STATUSES, + annotate: annotateRunOwners, + }; + case 'approvals-waiting': + // OPEN_STATUSES, not just 'pending': a claimed or escalated approval is + // still undecided, and its run is still suspended behind it. The same + // set the store's open-uniqueness index and decide()'s CAS ride on. + return { + key: ['id'], + detail: ['workflow_id', 'run_id', 'step_key', 'status', 'created_at'], + where: `status IN (${placeholdersFor(OPEN_STATUSES)})`, + binds: OPEN_STATUSES, + }; + case 'schedule-deferred-dispatches': + // 'deferred' is the ONLY outcome that is still work. The same table + // holds every settled fire under schedule-trigger retention, and a + // predicate that matched those would keep this category permanently + // non-empty and make the drain unprovable. The schedule store's own + // dispatch reads guard on exactly this value. + return { + key: ['id'], + detail: ['scheduleId', 'runId', 'scheduledFireAt', 'actualFireAt'], + where: "outcome = 'deferred'", + binds: [], + }; + case 'pending-notifications': { + const now = new Date(this.#now()).toISOString(); + // listDueNotifications' predicate, verbatim in meaning: pending AND + // (deliverAt or summaryAt has come due). A pending row that is not yet + // due is NOT work a drain can finish — no dispatch pass will select + // it — so it stays out of the page and is reported as `notDue` + // instead. That total also covers pending rows carrying NEITHER + // timestamp, which no dispatch pass will ever select at all. + const due = DUE_NOTIFICATION_SQL; + return { + key: ['thread_id', 'id'], + detail: ['source', 'kind', 'priority', 'agentId', 'deliverAt'], + where: due, + binds: [now, now], + // The aggregate scans every PENDING row so `notDue` can exist at + // all — but `count` re-applies the due predicate, so it still means + // what it means in every other category: rows in this one. + countWhere: "status = 'pending'", + countBinds: [], + countExpression: `SUM(CASE WHEN ${due} THEN 1 ELSE 0 END)`, + countExpressionBinds: [now, now], + totals: [`SUM(CASE WHEN ${due} THEN 0 ELSE 1 END) AS notDue`], + totalsBinds: [now, now], + }; + } + case 'background-tasks': + // Nonterminal is the complement of what the background-task TTL reaps, + // so a task this reports as outstanding is exactly a task retention + // will not touch. + return { + key: ['id'], + detail: [ + 'status', + 'tool_name', + 'agent_id', + 'run_id', + // The distinction that decides an operator's next move: a task + // parked BY THE FENCE resumes itself when the deployment reopens, + // while a task suspended by its own tool is waiting for something + // external and will still be waiting after the migration. + `${fenceParkedSql()} AS fenceSuspended`, + ], + flags: ['fenceSuspended'], + where: `status NOT IN (${placeholdersFor(BACKGROUND_TASK_TERMINAL_STATUSES)})`, + binds: BACKGROUND_TASK_TERMINAL_STATUSES, + totals: [`SUM(${fenceParkedSql()}) AS fenceSuspended`], + }; + case 'resource-owners': + // A non-null reservation token is an ownership claim that was taken and + // never settled — settleReservation is what clears it, and `owner()` + // deliberately reads only rows where it is already NULL. + // + // The token itself is never projected: it is the value a settle races + // on, and an inventory is a read, not a way to learn one. + return { + key: ['resource_kind', 'resource_id'], + detail: ['owner_kind', 'owner_id'], + where: 'reservation_token IS NOT NULL', + binds: [], + }; + case 'start-reservations': { + // Everything except 'terminal' — derived from the reservation state + // list rather than spelled, so a new state joins this category by + // default instead of vanishing from it. + const unsettled = START_RESERVATION_STATES.filter( + (state) => state !== 'terminal', + ); + return { + key: ['key'], + detail: [ + 'state', + 'owner_kind', + 'owner_id', + 'target_kind', + 'target_id', + 'run_id', + // The agent surface's address. Present for agent reservations only, + // and omitted rather than nulled for workflow ones — a retry of an + // agent key that cannot be routed to a thread is unreachable, so + // the field's absence is information. + 'thread_id', + 'updated_at', + ], + where: `state IN (${placeholdersFor(unsettled)})`, + binds: unsettled, + }; + } + case 'schedules': + // Standing configuration: every row, including paused ones. A paused + // schedule arms nothing today and everything the day it resumes, and a + // reconciling operator needs to see both. + return { + key: ['id'], + detail: [ + 'status', + 'nextFireAt', + `CASE WHEN json_valid(target) + THEN json_extract(target, '$.type') END AS targetType`, + `CASE WHEN json_valid(target) + THEN COALESCE( + json_extract(target, '$.workflowId'), + json_extract(target, '$.agentId') + ) END AS targetId`, + ], + where: '1 = 1', + binds: [], + }; + case 'signal-subscriptions': + return { + key: ['id'], + detail: [ + 'provider_id', + 'thread_id', + 'external_resource_id', + 'subscribed_at', + ], + where: '1 = 1', + binds: [], + }; + } + } +} diff --git a/packages/flowsafe/src/do-runner/mastra-schema-guard.test.ts b/packages/flowsafe/src/do-runner/mastra-schema-guard.test.ts index 9cbe867..301ae96 100644 --- a/packages/flowsafe/src/do-runner/mastra-schema-guard.test.ts +++ b/packages/flowsafe/src/do-runner/mastra-schema-guard.test.ts @@ -8,39 +8,79 @@ // MASTRA_TABLES names. A dependency bump that adds or renames one must make // its lifecycle decision explicit here. // -// The inventory is a STRUCTURE, not a name list (DL-003): each entry states -// whether the product adopts that table and how its rows expire. A -// @mastra/core bump changing the inventory must fail CI and reopen the -// persistence review, never silently ship. +// The inventory is a STRUCTURE, not a name list: each entry states whether +// the product adopts that table and how its rows expire. A @mastra/core bump +// changing the inventory must fail CI, reopen the persistence review, and +// never ship silently. // // 2. SCHEMA GUARD — retention jobs and app-owned indexes depend on Mastra's // column names and encodings. A @mastra/core bump renaming any of them must // fail here instead of silently disabling maintenance. +// +// 3. DRAIN-INVENTORY CENSUS — the same forcing function, aimed at a different +// question. Retention asks "what expires this row?"; the drain inventory +// (do-runner/inventory.ts) asks "does this row stop a migration?", and an +// operator reads an empty inventory as permission to lock a deployment and +// copy it. That permission is only as good as the claim that the inventory +// knows about every table, so every entry of MASTRA_TABLES and of +// FLOWSAFE_TABLES must name an inventory category or write down why it holds +// no drainable work — and the flowsafe half is cross-checked against the +// tables a fully-provisioned database actually contains, so a new +// `flowsafe_` table fails CI without anyone remembering to add it here. import type { MastraCompositeStore } from '@mastra/core/storage'; import { InMemoryStore } from '@mastra/core/storage'; import { describe, expect, it } from 'vitest'; import { z } from 'zod'; +import { + DEPLOYMENT_SENTINEL_TABLE, + EXECUTION_FENCE_TABLE, +} from '#deployment-identity-protocol'; import { openSqlite, type SqliteDatabase, sqliteUnitDatabase, } from '../../test-support/sqlite.js'; +import { + createResourceOwnershipSchema, + D1ApprovalStoreFactory, + RESOURCE_OWNERSHIP_TABLE, +} from '../approval-api/index.js'; +import { APPROVALS_TABLE } from '../approval-api/types.js'; import { createScheduleStorageDomains } from '../schedules/storage.js'; +import { + D1SubscriptionStoreFactory, + SIGNAL_SUBSCRIPTIONS_TABLE, +} from '../signal-providers/index.js'; +import { NOTIFICATION_SEQUENCE_TABLE } from '../signals/notifications-d1.js'; import { createSignalStorageDomains } from '../signals/storage.js'; import { BACKGROUND_TASK_TTL_PURGE_TABLES, createD1Storage, NOTIFICATION_TTL_PURGE_TABLES, + RESOURCE_OWNER_TABLE, RUN_TTL_PURGE_TABLES, SCHEDULE_TRIGGER_TTL_PURGE_TABLES, THREAD_STATE_TTL_PURGE_TABLES, THREAD_TTL_PURGE_TABLES, } from './d1-storage.js'; +import { ExecutionFenceStore } from './execution-fence.js'; import { init } from './init.js'; +import { + DeploymentInventory, + FLOWSAFE_TABLES, + INVENTORY_CATEGORIES, + INVENTORY_CATEGORY_DESCRIPTORS, + type InventoryDatabase, + type InventoryTableAccounting, +} from './inventory.js'; import { mintThreadId, resourceIdFromKey } from './memory-id.js'; import type { RunnerRuntime } from './runtime.js'; +import { + START_IDEMPOTENCY_TABLE, + StartIdempotencyStore, +} from './start-idempotency.js'; function tableNames(db: SqliteDatabase): string[] { return ( @@ -54,7 +94,10 @@ function tableNames(db: SqliteDatabase): string[] { // A gated workflow over the given storage. function buildGated(storage: MastraCompositeStore): RunnerRuntime { - const { createWorkflow, createStep, runtime } = init({ storage }); + const { createWorkflow, createStep, runtime } = init( + { storage }, + { startIdempotency: 'none', executionFence: 'none' }, + ); const gate = createStep({ id: 'gate', inputSchema: z.object({}), @@ -97,13 +140,22 @@ type StorageOwnership = 'deployment-wide' | 'unadopted'; type RetentionStory = | { kind: 'run-ttl' } | { kind: 'thread-ttl' } - /** Track B: rows expire via purgeExpiredBackgroundTasks (completedAt TTL). */ + /** + * Background-task rows expire via purgeExpiredBackgroundTasks (completedAt + * TTL). + */ | { kind: 'background-task-ttl' } - /** Track C: agent-inbox rows expire via purgeExpiredNotifications (terminal + updatedAt TTL). */ + /** + * Agent-inbox rows expire via purgeExpiredNotifications (terminal + updatedAt + * TTL). + */ | { kind: 'notification-ttl' } - /** Track C: thread-state rows expire via purgeExpiredThreadState (updatedAt TTL). */ + /** Thread-state rows expire via purgeExpiredThreadState (updatedAt TTL). */ | { kind: 'thread-state-ttl' } - /** Track D: schedule-trigger history expires via purgeExpiredScheduleTriggers (actualFireAt TTL). */ + /** + * Schedule-trigger history expires via purgeExpiredScheduleTriggers + * (actualFireAt TTL). + */ | { kind: 'schedule-trigger-ttl' } /** Rows die with their parent's TTL rather than aging out on their own. */ | { kind: 'cascade'; with: string } @@ -123,29 +175,48 @@ describe('Mastra persistence guards (D1Store SQL over node:sqlite)', () => { // biconditional below can tie them: an unadopted table has nothing to expire // BECAUSE nothing writes it, so it carries exactly this, and the day a track // becomes deployment-wide the tie forces retention off this - // boilerplate in the SAME change (the DL-003 "all three legs together" bar). + // boilerplate in the SAME change (the "all three legs together" bar). const UNADOPTED_NO_RETENTION = 'unadopted — no feature writes it, so there is nothing to expire yet'; + // The one reason an unadopted table gives the DRAIN inventory. Hoisted for + // the same purpose as UNADOPTED_NO_RETENTION above: it ties the exclusion to + // ownership, so the day a track starts writing one of these, both + // declarations have to be revisited in that change. + const UNADOPTED_NO_WORK = + 'unadopted — no feature writes it, so it can hold no outstanding work for a drain to wait on'; + const MASTRA_TABLES: ReadonlyArray<{ table: string; coverage: StorageOwnership; retention: RetentionStory; + /** + * Where this table's rows show up in the drain inventory, or why they never + * hold up a migration. Required, like `retention`: a Mastra bump that adds + * a table must answer BOTH questions in the change that adopts it. + */ + accounting: InventoryTableAccounting; }> = [ { table: 'mastra_background_tasks', coverage: 'deployment-wide', retention: { kind: 'background-task-ttl' }, + accounting: { category: 'background-tasks' }, }, { table: 'mastra_messages', coverage: 'deployment-wide', retention: { kind: 'cascade', with: 'mastra_threads' }, + accounting: { + excluded: + 'conversation history: a message records something that already happened and nothing executes it. Signals a draining deployment persists instead of waking land here too, and are declared unenumerable BECAUSE they are deliberately carried across the migration rather than drained.', + }, }, { table: 'mastra_notifications', coverage: 'deployment-wide', retention: { kind: 'notification-ttl' }, + accounting: { category: 'pending-notifications' }, }, { table: 'mastra_resources', @@ -155,8 +226,12 @@ describe('Mastra persistence guards (D1Store SQL over node:sqlite)', () => { because: "working memory is the owner's, shared across every thread they have, so one thread aging out says nothing about it; the resource is deleted explicitly with its owner", }, + accounting: { + excluded: + 'per-owner working memory: state the migration copies wholesale. It is never in flight, so there is nothing here for a drain to finish and no reading of it that could ever reach empty.', + }, }, - // Track D tables. Sorted: + // Schedule tables. Sorted: // 'mastra_schedule_triggers' precedes 'mastra_schedules' under BINARY // collation ('_' 0x5F < 's' 0x73, the same order 'mastra_thread_state' < // 'mastra_threads' takes), and both precede 'mastra_scorers' ('sch' < 'sco'). @@ -164,6 +239,7 @@ describe('Mastra persistence guards (D1Store SQL over node:sqlite)', () => { table: 'mastra_schedule_triggers', coverage: 'deployment-wide', retention: { kind: 'schedule-trigger-ttl' }, + accounting: { category: 'schedule-deferred-dispatches' }, }, { table: 'mastra_schedules', @@ -173,6 +249,7 @@ describe('Mastra persistence guards (D1Store SQL over node:sqlite)', () => { because: 'a schedule is standing configuration deleted explicitly; it has no terminal state to age out, while its fire history expires through schedule-trigger-ttl', }, + accounting: { category: 'schedules' }, }, { table: 'mastra_scorers', @@ -181,6 +258,7 @@ describe('Mastra persistence guards (D1Store SQL over node:sqlite)', () => { kind: 'none', because: UNADOPTED_NO_RETENTION, }, + accounting: { excluded: UNADOPTED_NO_WORK }, }, // 'mastra_thread_state' sorts BEFORE 'mastra_threads' under BINARY collation // ('_' 0x5F < 's' 0x73), which is the order sqlite_master's ORDER BY name @@ -189,21 +267,30 @@ describe('Mastra persistence guards (D1Store SQL over node:sqlite)', () => { table: 'mastra_thread_state', coverage: 'deployment-wide', retention: { kind: 'thread-state-ttl' }, + accounting: { + excluded: + "the agent's task list and its goal objective, one durable value per (thread, type). Both are standing state read on a thread's next turn — neither is queued, neither executes on its own, and neither carries a consumption marker a predicate could test.", + }, }, { table: 'mastra_threads', coverage: 'deployment-wide', retention: { kind: 'thread-ttl' }, + accounting: { + excluded: + 'thread identity and its state-signal tracking metadata. A thread is an ADDRESS, not work: the runs addressed to it are inventoried under `runs`, and a thread with no live run owes a migration nothing.', + }, }, { table: 'mastra_workflow_snapshot', coverage: 'deployment-wide', retention: { kind: 'run-ttl' }, + accounting: { category: 'runs' }, }, ]; it('createD1Storage creates exactly the declared deployment inventory', async () => { - // #given — the real storage adapter over sqlite, WITH the Track C signal + // #given — the real storage adapter over sqlite, WITH the signal // domains composed (createD1Storage takes them injected — signals/ imports // do-runner, so do-runner cannot import back). @mastra/cloudflare-d1 ships no // notifications/thread-state domain, so those two tables come from the @@ -326,7 +413,202 @@ describe('Mastra persistence guards (D1Store SQL over node:sqlite)', () => { } }); - it('the Track D schedule tables keep the columns their purge + tick ride on (metadata + nextFireAt/actualFireAt)', async () => { + // ------------------------------------------------------------------------- + // The drain-inventory census + // ------------------------------------------------------------------------- + + /** Both halves of the census as one list, which is how it is asked about. */ + const CENSUS: ReadonlyArray<{ + table: string; + accounting: InventoryTableAccounting; + }> = [ + ...MASTRA_TABLES.map((entry) => ({ + table: entry.table, + accounting: entry.accounting, + })), + ...FLOWSAFE_TABLES.map((entry) => ({ + table: entry.table, + accounting: entry.accounting, + })), + ]; + + /** + * A database with every table its real owner would create — the adapter's, + * the flowsafe domains', the approval and ownership registries', the + * reservation store's, the subscription factory's, and the two the + * provisioning protocol writes. + */ + async function fullyProvisioned(): Promise<{ + sqlite: SqliteDatabase; + binding: unknown; + }> { + const sqlite = openSqlite(); + const binding = sqliteUnitDatabase(sqlite); + const storage = createD1Storage({ + binding: binding as never, + domains: { + ...createSignalStorageDomains(binding as never), + ...createScheduleStorageDomains(binding as never), + }, + }); + const runtime = buildGated(storage); + await runtime.start('gated', { runId: 'abc_r1', inputData: {} }); + await storage.init(); + await createResourceOwnershipSchema(binding as never); + // The approval store creates its schema lazily on first use; a create is + // the cheapest way to make it happen without hand-writing its DDL here. + await new D1ApprovalStoreFactory(binding as never).store().create({ + id: 'apr-census', + workflowId: 'gated', + runId: 'abc_r1', + title: 'census', + connectors: [], + priority: 'normal', + status: 'pending', + createdAt: new Date(0).toISOString(), + updatedAt: new Date(0).toISOString(), + }); + await new StartIdempotencyStore(binding as never).reserve({ + key: 'census-key', + owner: { kind: 'human', id: 'ada' }, + targetKind: 'workflow', + targetId: 'gated', + mintRunId: () => 'abc_r9', + }); + await new D1SubscriptionStoreFactory(binding as never, { + uuid: () => 'sub-census', + }) + .store() + .subscribe({ + providerId: 'github', + externalResourceId: 'octo/repo#1', + threadId: 'thr-1', + resourceId: 'res-1', + }); + await new ExecutionFenceStore(binding as never).seed('open'); + sqlite.exec( + `CREATE TABLE IF NOT EXISTS ${DEPLOYMENT_SENTINEL_TABLE} ( + id INTEGER PRIMARY KEY CHECK (id = 1), + tenant_tag TEXT NOT NULL, + provisioned_at TEXT NOT NULL + )`, + ); + return { sqlite, binding }; + } + + it('every censused table names an inventory category or writes down why a drain may ignore it', async () => { + // #given — the census is what makes an empty inventory MEAN anything. A + // table in neither list is a table an operator would never be shown and + // would still be migrating away from. + for (const entry of CENSUS) { + // #then — exactly one of the two arms, and a category that really exists. + if ('category' in entry.accounting) { + expect( + INVENTORY_CATEGORIES as readonly string[], + `${entry.table} claims a category the inventory does not serve`, + ).toContain(entry.accounting.category); + continue; + } + // #then — an exclusion is a SENTENCE. The type can demand the key but not + // the content, and "not work" is indistinguishable from "nobody looked" + // at that length — the same bar the retention reasons are held to. + expect( + entry.accounting.excluded.trim().length, + `${entry.table} is excluded from the inventory with an empty/blank reason`, + ).toBeGreaterThan(20); + } + }); + + it('every inventory category is claimed by exactly one censused table, and every category names a real reader over it', async () => { + // #given — the other direction: a category nothing feeds is a promise the + // index makes and no query keeps. + const claimed = CENSUS.flatMap((entry) => + 'category' in entry.accounting ? [entry.accounting.category] : [], + ); + + // #then — a bijection between categories and the tables that claim them. + expect([...claimed].sort()).toEqual([...INVENTORY_CATEGORIES].sort()); + expect(new Set(claimed).size).toBe(claimed.length); + + // #then — and each descriptor's declared table is the table the reader + // really queries. A descriptor naming a table its SQL does not read would + // report an empty category forever while the index insisted it was covered. + const { binding } = await fullyProvisioned(); + for (const descriptor of INVENTORY_CATEGORY_DESCRIPTORS) { + const statements: string[] = []; + const inner = binding as InventoryDatabase; + const inventory = new DeploymentInventory({ + prepare(query: string) { + statements.push(query); + return inner.prepare(query); + }, + }); + await inventory.read(descriptor.category); + expect( + statements.some((sql) => + new RegExp(`FROM\\s+${descriptor.table}\\b`).test(sql), + ), + `${descriptor.category} declares table ${descriptor.table} but reads something else`, + ).toBe(true); + // #then — and the table it declares is one the census accounts for. + expect(CENSUS.map((entry) => entry.table)).toContain(descriptor.table); + } + }); + + it('the flowsafe census matches the flowsafe_ tables a provisioned deployment actually has', async () => { + // #given — the mastra_% inventory above catches a @mastra/core bump. This + // is its flowsafe-owned half, and it is the leg that makes the census + // self-maintaining: a new flowsafe table fails CI on the day it is created, + // whether or not the author remembered this file. + const { sqlite } = await fullyProvisioned(); + + // #when + const present = ( + sqlite + .prepare( + `SELECT name FROM sqlite_master + WHERE type='table' AND name LIKE 'flowsafe_%' ORDER BY name`, + ) + .all() as Array<{ name: string }> + ).map((row) => row.name); + + // #then — exact equality both ways: an uncensused table fails, and so does + // a censused one nothing creates any more. + expect(present).toEqual([...FLOWSAFE_TABLES.map((e) => e.table)].sort()); + }); + + it('every table name the census and the inventory restate is the one its owner declares', async () => { + // #given — three names live in two places, because the layering forbids the + // import: do-runner may not reach the ownership store, the subscription + // registry, or the approval store, so the copies below are unavoidable. + // What is avoidable is a rename that silently empties a category, and this + // is where the two sides can finally be compared. + const censused = FLOWSAFE_TABLES.map((entry) => entry.table); + + // #then + expect(RESOURCE_OWNER_TABLE).toBe(RESOURCE_OWNERSHIP_TABLE); + expect(censused).toContain(RESOURCE_OWNERSHIP_TABLE); + expect(censused).toContain(SIGNAL_SUBSCRIPTIONS_TABLE); + expect(censused).toContain(APPROVALS_TABLE); + expect(censused).toContain(START_IDEMPOTENCY_TABLE); + expect(censused).toContain(NOTIFICATION_SEQUENCE_TABLE); + expect(censused).toContain(EXECUTION_FENCE_TABLE); + expect(censused).toContain(DEPLOYMENT_SENTINEL_TABLE); + + // #then — and every mastra_ table the inventory declares is one the storage + // inventory above pins, so a typo in a restated name fails here rather than + // becoming a category that is empty forever. + const mastraTables = MASTRA_TABLES.map((entry) => entry.table); + for (const descriptor of INVENTORY_CATEGORY_DESCRIPTORS) { + if (!descriptor.table.startsWith('mastra_')) continue; + expect( + mastraTables, + `${descriptor.category} names ${descriptor.table}, which createD1Storage does not create`, + ).toContain(descriptor.table); + } + }); + + it('the schedule tables keep the columns their purge + tick ride on (metadata + nextFireAt/actualFireAt)', async () => { // #given — flowsafe-owned tables (the adapter ships neither); compose the // schedules domain and init to create them. const sqlite = openSqlite(); @@ -419,8 +701,8 @@ describe('Mastra persistence guards (D1Store SQL over node:sqlite)', () => { expect(columns).toContain('workflow_name'); }); - it('mastra_background_tasks keeps the columns Track B purges ride on (run_id range + completedAt/status TTL)', async () => { - // #given — the real adapter creates the table eagerly (Track B adopted it) + it('mastra_background_tasks keeps the columns its purges ride on (run_id range + completedAt/status TTL)', async () => { + // #given — the real adapter creates the table eagerly const sqlite = openSqlite(); const storage = createD1Storage({ binding: sqliteUnitDatabase(sqlite) as never, @@ -442,7 +724,7 @@ describe('Mastra persistence guards (D1Store SQL over node:sqlite)', () => { expect(columns).toContain('completedAt'); }); - it('the Track C signal tables keep the columns their purges ride on (thread_id range + status/updatedAt TTL)', async () => { + it('the signal tables keep the columns their purges ride on (thread_id range + status/updatedAt TTL)', async () => { // #given — these two tables are flowsafe-owned (the adapter ships neither), // so compose the domains and init to create them. const sqlite = openSqlite(); diff --git a/packages/flowsafe/src/do-runner/notification-predicate.ts b/packages/flowsafe/src/do-runner/notification-predicate.ts new file mode 100644 index 0000000..3e67e3a --- /dev/null +++ b/packages/flowsafe/src/do-runner/notification-predicate.ts @@ -0,0 +1,10 @@ +// SPDX-License-Identifier: Apache-2.0 + +/** + * Pending notifications whose delivery or summary time has arrived. + * + * Bind two positional parameters to the same ISO "now": `deliverAt` first, + * then `summaryAt`. + */ +export const DUE_NOTIFICATION_SQL = + "status = 'pending' AND ((deliverAt IS NOT NULL AND deliverAt <= ?) OR (summaryAt IS NOT NULL AND summaryAt <= ?))"; diff --git a/packages/flowsafe/src/do-runner/runtime.test.ts b/packages/flowsafe/src/do-runner/runtime.test.ts index 41220a2..4c34a9a 100644 --- a/packages/flowsafe/src/do-runner/runtime.test.ts +++ b/packages/flowsafe/src/do-runner/runtime.test.ts @@ -52,7 +52,10 @@ function buildRuntime(storage: InMemoryStore): { counters: Counters; } { const counters: Counters = { approvalResumes: 0, echoRuns: 0 }; - const { createWorkflow, createStep, runtime } = init({ storage }); + const { createWorkflow, createStep, runtime } = init( + { storage }, + { startIdempotency: 'none', executionFence: 'none' }, + ); const research = createStep({ id: 'research', @@ -118,18 +121,24 @@ describe('RunnerRuntime host pubsub identity', () => { const pubsub = createHostPubSub(); // #when — init() threads it (InitOptions.pubsub -> RunnerRuntimeOptions.pubsub) - const { runtime } = init({ storage: new InMemoryStore() }, { pubsub }); + const { runtime } = init( + { storage: new InMemoryStore() }, + { startIdempotency: 'none', pubsub, executionFence: 'none' }, + ); - // #then — the SAME instance is reachable, so Track A's createRun sites and - // observe() replay share one feed. Delete the thread in init.ts and this - // fails: runtime.pubsub is undefined and the two createRun sites would each - // let core default a separate emitter — the DL-001 bug this seam prevents. + // #then — the SAME instance is reachable, so the agent's createRun + // sites and observe() replay share one feed. Delete the thread in init.ts + // and this fails: runtime.pubsub is undefined, so the two createRun sites + // each let core default a separate emitter — the bug this seam prevents. expect(runtime.pubsub).toBe(pubsub); }); it('leaves runtime.pubsub undefined when the host configures none (byte-identical)', () => { // #when — no pubsub passed - const { runtime } = init({ storage: new InMemoryStore() }); + const { runtime } = init( + { storage: new InMemoryStore() }, + { startIdempotency: 'none', executionFence: 'none' }, + ); // #then — undefined, the polling-fallback posture existing hosts keep expect(runtime.pubsub).toBeUndefined(); @@ -138,9 +147,10 @@ describe('RunnerRuntime host pubsub identity', () => { describe('RunnerRuntime', () => { it('passes initial workflow state through to core execution', async () => { - const { createWorkflow, createStep, runtime } = init({ - storage: new InMemoryStore(), - }); + const { createWorkflow, createStep, runtime } = init( + { storage: new InMemoryStore() }, + { startIdempotency: 'none', executionFence: 'none' }, + ); const stateSchema = z.object({ seed: z.string() }); const inspect = createStep({ id: 'inspect-state', @@ -572,7 +582,10 @@ describe('RunnerRuntime', () => { it('rejects duplicate workflow ids at registration', () => { // #given - const { createWorkflow } = init({ storage: new InMemoryStore() }); + const { createWorkflow } = init( + { storage: new InMemoryStore() }, + { startIdempotency: 'none', executionFence: 'none' }, + ); createWorkflow({ id: 'wf', inputSchema: z.object({}), @@ -591,7 +604,10 @@ describe('RunnerRuntime', () => { it('rejects duplicate agent ids at registration', () => { // #given - const { runtime } = init({ storage: new InMemoryStore() }); + const { runtime } = init( + { storage: new InMemoryStore() }, + { startIdempotency: 'none', executionFence: 'none' }, + ); runtime.registerAgent(runtimeAgent('writer')); // #when / #then @@ -602,9 +618,10 @@ describe('RunnerRuntime', () => { it('exposes every registered agent to workflow execution', async () => { // #given - const { createWorkflow, createStep, runtime } = init({ - storage: new InMemoryStore(), - }); + const { createWorkflow, createStep, runtime } = init( + { storage: new InMemoryStore() }, + { startIdempotency: 'none', executionFence: 'none' }, + ); const agents = [ runtimeAgent('a'), runtimeAgent('b'), @@ -669,7 +686,7 @@ describe('RunnerRuntime', () => { const publish = vi.spyOn(pubsub, 'publish'); const { createWorkflow, createStep, runtime } = init( { storage: new InMemoryStore() }, - { pubsub }, + { startIdempotency: 'none', pubsub, executionFence: 'none' }, ); const agent = runtimeAgent('writer'); runtime.registerAgent(agent); @@ -710,9 +727,10 @@ describe('RunnerRuntime', () => { 'constructor', ])("executes prototype-collision workflow id '%s'", async (workflowId) => { // #given - const { createWorkflow, createStep, runtime } = init({ - storage: new InMemoryStore(), - }); + const { createWorkflow, createStep, runtime } = init( + { storage: new InMemoryStore() }, + { startIdempotency: 'none', executionFence: 'none' }, + ); const noop = createStep({ id: 'noop', inputSchema: z.object({}), @@ -750,7 +768,10 @@ describe('RunnerRuntime', () => { '', ])("rejects non-path-safe workflow id '%s' at registration", (id) => { // #given - const { createWorkflow } = init({ storage: new InMemoryStore() }); + const { createWorkflow } = init( + { storage: new InMemoryStore() }, + { startIdempotency: 'none', executionFence: 'none' }, + ); // #when / #then — a ':' or '/' in the id would make the DO name join // (`${workflowId}:${runId}`) and the /runs/:workflowId/:runId path @@ -766,7 +787,10 @@ describe('RunnerRuntime', () => { it('accepts path-safe workflow ids at registration', () => { // #given - const { createWorkflow, runtime } = init({ storage: new InMemoryStore() }); + const { createWorkflow, runtime } = init( + { storage: new InMemoryStore() }, + { startIdempotency: 'none', executionFence: 'none' }, + ); // #when — unreserved-character id, including the '.' that only the bare // dot-segments '.' and '..' are barred from @@ -782,9 +806,10 @@ describe('RunnerRuntime', () => { it('rejects registration after the first run', async () => { // #given - const { createWorkflow, createStep, runtime } = init({ - storage: new InMemoryStore(), - }); + const { createWorkflow, createStep, runtime } = init( + { storage: new InMemoryStore() }, + { startIdempotency: 'none', executionFence: 'none' }, + ); const step = createStep({ id: 'noop', inputSchema: z.object({}), @@ -819,7 +844,10 @@ const ISO_8601 = /^\d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2}/; describe('RunnerRuntime.status projection', () => { it('persists a terminal status when a workflow retains only resume snapshots', async () => { const storage = new InMemoryStore(); - const { createWorkflow, createStep, runtime } = init({ storage }); + const { createWorkflow, createStep, runtime } = init( + { storage }, + { startIdempotency: 'none', executionFence: 'none' }, + ); const gate = createStep({ id: 'gate', inputSchema: z.object({ value: z.string() }), @@ -868,7 +896,10 @@ describe('RunnerRuntime.status projection', () => { it('persists a terminal start when the engine omits terminal snapshots', async () => { const storage = new InMemoryStore(); - const { createWorkflow, createStep, runtime } = init({ storage }); + const { createWorkflow, createStep, runtime } = init( + { storage }, + { startIdempotency: 'none', executionFence: 'none' }, + ); const echo = createStep({ id: 'echo-once', inputSchema: z.object({ value: z.string() }), @@ -945,9 +976,10 @@ describe('RunnerRuntime.status projection', () => { it('projects the failure message for a failed run', async () => { // #given - const { createWorkflow, createStep, runtime } = init({ - storage: new InMemoryStore(), - }); + const { createWorkflow, createStep, runtime } = init( + { storage: new InMemoryStore() }, + { startIdempotency: 'none', executionFence: 'none' }, + ); const boom = createStep({ id: 'boom', inputSchema: z.object({}), @@ -982,9 +1014,10 @@ describe('RunnerRuntime.status projection', () => { // object, not an Error instance. This is the shape errorText() defends // against at an engine/persistence boundary; String() on it reads // '[object Object]', so a naive projection would lose the message. - const { createWorkflow, createStep, runtime } = init({ - storage: new InMemoryStore(), - }); + const { createWorkflow, createStep, runtime } = init( + { storage: new InMemoryStore() }, + { startIdempotency: 'none', executionFence: 'none' }, + ); const boom = createStep({ id: 'boom-object', inputSchema: z.object({}), @@ -1022,9 +1055,10 @@ describe('RunnerRuntime.status projection', () => { it('projects every branch of a multi-step (parallel) suspension', async () => { // #given — two parallel steps that both suspend in the same run - const { createWorkflow, createStep, runtime } = init({ - storage: new InMemoryStore(), - }); + const { createWorkflow, createStep, runtime } = init( + { storage: new InMemoryStore() }, + { startIdempotency: 'none', executionFence: 'none' }, + ); const makeGate = (id: string, reason: string) => createStep({ id, @@ -1087,9 +1121,10 @@ describe('RunnerRuntime run lifecycle', () => { const held = new Promise((resolve) => { release = resolve; }); - const { createWorkflow, createStep, runtime } = init({ - storage: options.storage ?? new InMemoryStore(), - }); + const { createWorkflow, createStep, runtime } = init( + { storage: options.storage ?? new InMemoryStore() }, + { startIdempotency: 'none', executionFence: 'none' }, + ); const step = createStep({ id: 'held', inputSchema: z.object({}), @@ -1181,6 +1216,8 @@ describe('RunnerRuntime run lifecycle', () => { const { createWorkflow, createStep, runtime } = init( { storage }, { + startIdempotency: 'none', + executionFence: 'none', requestContextForRun: () => ({ 'flowsafe.runLifecycle': { version: 1, @@ -1873,7 +1910,10 @@ describe('RunnerRuntime run lifecycle', () => { it('serializes terminal reconciliation with a concurrent cancellation preflight', async () => { const storage = new InMemoryStore(); - const { createWorkflow, createStep, runtime } = init({ storage }); + const { createWorkflow, createStep, runtime } = init( + { storage }, + { startIdempotency: 'none', executionFence: 'none' }, + ); const step = createStep({ id: 'finish', inputSchema: z.object({}), @@ -1981,7 +2021,11 @@ describe('RunnerRuntime requestContextForRun', () => { const seen: Observation[] = []; const { createWorkflow, createStep, runtime } = init( { storage: new InMemoryStore() }, - { requestContextForRun: provider }, + { + startIdempotency: 'none', + requestContextForRun: provider, + executionFence: 'none', + }, ); const first = createStep({ id: 'first', @@ -2027,7 +2071,11 @@ describe('RunnerRuntime requestContextForRun', () => { const seen: Record = {}; const { createWorkflow, createStep, runtime } = init( { storage: new InMemoryStore() }, - { requestContextForRun: () => ({ 'test.a': 'provider' }) }, + { + startIdempotency: 'none', + requestContextForRun: () => ({ 'test.a': 'provider' }), + executionFence: 'none', + }, ); const inspect = createStep({ id: 'inspect-scheduled-context', @@ -2188,9 +2236,10 @@ describe('RunnerRuntime requestContextForRun', () => { // #given — NO requestContextForRun provider; a step that records the // runtime-minted scope (breakwater's crossWorkflowIsolation reads it) const seen: unknown[] = []; - const { createWorkflow, createStep, runtime } = init({ - storage: new InMemoryStore(), - }); + const { createWorkflow, createStep, runtime } = init( + { storage: new InMemoryStore() }, + { startIdempotency: 'none', executionFence: 'none' }, + ); const probe = createStep({ id: 'probe', inputSchema: z.object({}), @@ -2221,9 +2270,10 @@ describe('RunnerRuntime requestContextForRun', () => { it('does not synthesize breakwater isolation scope from runId prefixes', async () => { // #given — a probe recording both server-minted keys const seen: Array<{ scope: unknown; isolation: unknown }> = []; - const { createWorkflow, createStep, runtime } = init({ - storage: new InMemoryStore(), - }); + const { createWorkflow, createStep, runtime } = init( + { storage: new InMemoryStore() }, + { startIdempotency: 'none', executionFence: 'none' }, + ); const probe = createStep({ id: 'probe', inputSchema: z.object({}), @@ -2263,6 +2313,8 @@ describe('RunnerRuntime requestContextForRun', () => { const { createWorkflow, createStep, runtime } = init( { storage: new InMemoryStore() }, { + startIdempotency: 'none', + executionFence: 'none', requestContextForRun: () => ({ 'breakwater.workflowScope': 'overridden', }), @@ -2300,6 +2352,8 @@ describe('RunnerRuntime requestContextForRun', () => { const { createWorkflow, createStep, runtime } = init( { storage: new InMemoryStore() }, { + startIdempotency: 'none', + executionFence: 'none', requestContextForRun: () => ({ stored: 'kept', 'breakwater.workflowScope': 'forged-workflow', @@ -2413,14 +2467,22 @@ describe('RunnerRuntime resumeCount projection (re-suspension)', () => { let rounds = 0; const { createWorkflow, createStep, runtime } = init( { storage }, - onLeg - ? { - requestContextForRun: (_workflowId, _runId, leg) => { - onLeg(leg); - return undefined; - }, - } - : undefined, + { + startIdempotency: 'none', + executionFence: 'none', + ...(onLeg + ? { + requestContextForRun: ( + _workflowId: string, + _runId: string, + leg: RunLeg, + ) => { + onLeg(leg); + return undefined; + }, + } + : {}), + }, ); const gate2x = createStep({ id: 'gate2x', @@ -2557,9 +2619,10 @@ describe('RunnerRuntime resumeCount projection (re-suspension)', () => { // optional-schema / validateInputs-off steps. If a Mastra bump changes // this, the "schema-less fixture required" assumption (buildReSuspender, // the relaunch-falsy e2e fixture) goes silently stale. - const { createWorkflow, createStep, runtime } = init({ - storage: new InMemoryStore(), - }); + const { createWorkflow, createStep, runtime } = init( + { storage: new InMemoryStore() }, + { startIdempotency: 'none', executionFence: 'none' }, + ); const schemaGate = createStep({ id: 'schemaGate', inputSchema: z.object({}), @@ -2634,9 +2697,10 @@ describe('RunnerRuntime resumeCount projection (re-suspension)', () => { it('marks only the resumed branch, leaving a co-suspended branch a first suspension', async () => { // #given — two parallel gates both suspend; gateA re-suspends on a payload // resume (round 2), gateB stays at its first suspension. - const { createWorkflow, createStep, runtime } = init({ - storage: new InMemoryStore(), - }); + const { createWorkflow, createStep, runtime } = init( + { storage: new InMemoryStore() }, + { startIdempotency: 'none', executionFence: 'none' }, + ); let aRounds = 0; const gateA = createStep({ id: 'gateA', @@ -2704,14 +2768,22 @@ describe('RunnerRuntime resumeCount snapshot provenance (shared runId across wor let bRounds = 0; const { createWorkflow, createStep, runtime } = init( { storage: new InMemoryStore() }, - onLeg - ? { - requestContextForRun: (workflowId, _runId, leg) => { - onLeg(workflowId, leg); - return undefined; - }, - } - : undefined, + { + startIdempotency: 'none', + executionFence: 'none', + ...(onLeg + ? { + requestContextForRun: ( + workflowId: string, + _runId: string, + leg: RunLeg, + ) => { + onLeg(workflowId, leg); + return undefined; + }, + } + : {}), + }, ); const gateA = createStep({ id: 'gate', @@ -2814,9 +2886,10 @@ describe('RunnerRuntime resumeCount snapshot provenance (shared runId across wor // suspended at any depth and status() keeps projecting its accumulating // ordinal — the deep-chain (3+ suspension) case the pair-binding relies on. function buildSharedDeepChain(): RunnerRuntime { - const { createWorkflow, createStep, runtime } = init({ - storage: new InMemoryStore(), - }); + const { createWorkflow, createStep, runtime } = init( + { storage: new InMemoryStore() }, + { startIdempotency: 'none', executionFence: 'none' }, + ); for (const id of ['wfA', 'wfB']) { const gate = createStep({ id: 'gate', @@ -2878,6 +2951,8 @@ describe('RunnerRuntime snapshot provenance durability', () => { const { createWorkflow, createStep, runtime } = init( { storage }, { + startIdempotency: 'none', + executionFence: 'none', requestContextForRun: (_workflowId, _runId, leg) => { onLeg?.(leg); return providedContext; @@ -2908,7 +2983,11 @@ describe('RunnerRuntime snapshot provenance durability', () => { ): RunnerRuntime { const { createWorkflow, createStep, runtime } = init( { storage }, - { requestContextForRun }, + { + startIdempotency: 'none', + requestContextForRun, + executionFence: 'none', + }, ); const gate = createStep({ id: 'gate', @@ -3210,7 +3289,10 @@ describe('per-suspension deadline contract', () => { start: () => Promise; } { const storage = new InMemoryStore(); - const { createWorkflow, createStep, runtime } = init({ storage }); + const { createWorkflow, createStep, runtime } = init( + { storage }, + { startIdempotency: 'none', executionFence: 'none' }, + ); // Built as a value so the reserved key survives a suspendSchema that does // not declare it — which is exactly what the stripping test measures. const suspendPayload: Record = { @@ -3405,9 +3487,10 @@ describe('per-suspension deadline contract', () => { // payload and the fence by the TOP-LEVEL step, so there is no fence for // the step that actually suspended. v1 refuses it, loudly, rather than // leaving an author to believe a deadline was accepted. - const { createWorkflow, createStep, runtime } = init({ - storage: new InMemoryStore(), - }); + const { createWorkflow, createStep, runtime } = init( + { storage: new InMemoryStore() }, + { startIdempotency: 'none', executionFence: 'none' }, + ); const approval = createStep({ id: 'approval', inputSchema: z.object({}), @@ -3488,9 +3571,10 @@ describe('per-suspension deadline contract', () => { // differently for a dotted id — ['a.b'] live, ['a','b'] rehydrated — and // an entry derived from one that the other cannot recognize is a deadline // that arms and then silently disappears. - const { createWorkflow, createStep, runtime } = init({ - storage: new InMemoryStore(), - }); + const { createWorkflow, createStep, runtime } = init( + { storage: new InMemoryStore() }, + { startIdempotency: 'none', executionFence: 'none' }, + ); const gate = createStep({ id: stepId, inputSchema: z.object({}), diff --git a/packages/flowsafe/src/do-runner/runtime.ts b/packages/flowsafe/src/do-runner/runtime.ts index 082ac4f..2e468a8 100644 --- a/packages/flowsafe/src/do-runner/runtime.ts +++ b/packages/flowsafe/src/do-runner/runtime.ts @@ -51,6 +51,12 @@ import { RUN_PROVENANCE_CONTEXT_KEY, stripReservedExecutionContext, } from './execution-context.js'; +import { + admitsExistingRun, + admitsRunStart, + ExecutionFencedError, + type ExecutionFenceStore, +} from './execution-fence.js'; import { mastraRegistryEntries } from './mastra-registry.js'; import { isPathSafeId } from './path-safe-id.js'; import type { HostPubSub } from './pubsub.js'; @@ -68,6 +74,7 @@ import { type RunTerminalErrorEnvelope, type RunTerminalStatus, } from './run-lifecycle.js'; +import type { StartIdempotencyStore } from './start-idempotency.js'; export class UnknownWorkflowError extends Error { constructor(workflowId: string) { @@ -739,6 +746,39 @@ export interface RunnerRuntimeOptions { * before this seam existed. */ pubsub?: HostPubSub; + /** + * The deployment execution fence (execution-fence.ts), consulted on EVERY + * start and resume. THIS is the closure guarantee for runs: every mint in + * this package funnels through start() and every re-entry through resume(), + * so a check here cannot be routed around by a surface that forgot to gate + * itself — the route-level checks are the same refusal made earlier and + * cheaper, never the boundary. + * + * Absent ⇒ unfenced, byte-identical to before this seam existed. `init()` + * builds one automatically from a `{ DB }` source and REQUIRES an explicit + * `executionFence` (a store, or `'none'`) from a `{ storage }` one, so + * absence here is always something a host wrote down. + */ + executionFence?: ExecutionFenceStore; + /** + * The deployment's start reservations (start-idempotency.ts). The runtime + * neither creates nor claims one — the surfaces above it do — but it is the + * one layer that sees EVERY way a run reaches a terminal state, so it owns + * the terminal reconcile that marks a key spent. + * + * Homed here rather than at the routes for exactly that reason: a run can end + * by completing, by failing, by being cancelled and by timing out, on the + * workflow surface and the agent surface alike, and a reconcile attached to + * any one route would miss the rest. A reservation that never settles is not + * a correctness bug (replay still answers from the snapshot) but it never + * leaves the drain inventory and never becomes purgeable, so a deployment + * would eventually be unable to prove itself empty. + * + * Absent ⇒ no reconcile. `init()` builds one from a `{ DB }` source, and + * `DurableObjectRunner.build` refuses a runtime that has none while a DB is + * bound, so absence means a host with no database to reserve against. + */ + startIdempotency?: StartIdempotencyStore; } /** @inline */ @@ -779,6 +819,20 @@ export type StartRunOptions = { economicOperations?: readonly RunEconomicOperation[]; /** Trusted schedule source; never accepted directly from a public request. */ scheduleDispatch?: RunScheduleDispatch; + /** + * The start's idempotency key. The runtime uses it for exactly one thing: the + * execution fence's proof-only state admits the start whose key matches its + * nominated proof key, and this is where that comparison happens. The + * exactly-once property the key also carries is enforced ABOVE the runtime, + * by the start reservation the surfaces take before calling in. + * + * INTERNAL: it reaches the runtime from a trusted host seam, never from a + * request body and never through an open request-context key — a tenant able + * to name the proof key could start a run on a deployment that is supposed to + * be executing exactly one. + * @internal + */ + idempotencyKey?: string; } & OptionalRunRequester; export type ResumeRunOptions = { @@ -907,10 +961,12 @@ export class RunnerRuntime { readonly #terminalAbortIntents = new Map(); readonly #lifecycleLocks = new Map>(); // The host DO's pubsub identity (RunnerRuntimeOptions.pubsub), threaded into - // both createRun sites below (CI-M-002-002) so a configured host publishes and + // both createRun sites below so a configured host publishes and // replays on ONE shared feed. Undefined ⇒ core defaults a fresh emitter per // run ⇒ byte-identical to before this seam existed. readonly #pubsub?: HostPubSub; + readonly #executionFence?: ExecutionFenceStore; + readonly #startIdempotency?: StartIdempotencyStore; #mastra?: Mastra; constructor(options: RunnerRuntimeOptions) { @@ -918,6 +974,51 @@ export class RunnerRuntime { this.#logger = options.logger ?? false; this.#requestContextForRun = options.requestContextForRun; this.#pubsub = options.pubsub; + this.#executionFence = options.executionFence; + this.#startIdempotency = options.startIdempotency; + } + + /** + * The deployment execution fence this runtime enforces, or undefined when + * the host built an unfenced runtime. Exposed so the surfaces ABOVE the + * runtime — the run object's routes, the thread DO's signal routes — gate on + * the same store rather than constructing a second one, and so + * DurableObjectRunner can assert that a runtime built against a bound + * database is never fence-less. + */ + get executionFence(): ExecutionFenceStore | undefined { + return this.#executionFence; + } + + /** + * The deployment's start reservations, or undefined for a host with no + * database to reserve against. Exposed for the same reason the fence is: the + * surfaces ABOVE this runtime reserve and claim against it, and two stores + * over two bindings would be two different tables answering the same key. + */ + get startIdempotency(): StartIdempotencyStore | undefined { + return this.#startIdempotency; + } + + /** + * Is this run EXECUTING in this isolate right now? + * + * The liveness half of the idempotent-start replay decision, answered from + * the same `#activeRuns` map the cancel path uses — the runtime's own record + * of runs it is currently driving. It is in-memory ON PURPOSE: liveness is a + * property of an isolate that is running code, and any durable proxy for it + * (a journal, a heartbeat, a timestamp) would keep saying "live" after the + * isolate that wrote it was evicted, which is precisely the case the probe + * exists to detect. + * + * A `false` here therefore means "not running HERE", which is authoritative + * only where the run has exactly one possible host — a run Durable Object + * addressed by `idFromName(workflowId:runId)`, or the thread object an agent + * run is bound to. Callers that probe across an object boundary must ask the + * object that owns the run, never their own runtime. + */ + isRunActive(workflowId: string, runId: string): boolean { + return this.#activeRuns.has(this.#runKey(workflowId, runId)); } /** @@ -972,6 +1073,69 @@ export class RunnerRuntime { return [...this.#workflows.keys()]; } + /** + * The fence check every mint passes. In proof-only the admitted start also + * BINDS the proof to its run id, and that write-back is conditional: between + * the read that admitted it and the write another start may have claimed the + * proof, or the operator may have moved the fence on. Zero rows changed + * therefore refuses the start — the deployment is no longer the one this + * start read. + */ + async #assertStartFence( + runId: string, + idempotencyKey: string | undefined, + ): Promise { + const fence = this.#executionFence; + if (!fence) return; + const reading = await fence.read(); + if (!admitsRunStart(reading, idempotencyKey)) { + throw new ExecutionFencedError(reading.state, 'run start'); + } + if (reading.state !== 'proof-only' || reading.proofKey === undefined) { + return; + } + if (!(await fence.recordProofRun(reading.proofKey, runId))) { + throw new ExecutionFencedError(reading.state, 'run start'); + } + } + + /** + * Mark this run's start reservation spent, if it has one. + * + * BEST EFFORT, and deliberately so: the run has already reached a terminal + * state and its snapshot is already persisted, so failing the caller here + * would turn a completed run into an error response — while the reconcile it + * failed to make costs only the LATER answer's precision (a purged run + * replays as UNRESOLVABLE rather than ALREADY_SETTLED, which refuses either + * way). The retention purge marks any reservation this missed, so a swallowed + * failure heals rather than accumulating. + */ + async #settleStartReservation(runId: string): Promise { + const store = this.#startIdempotency; + if (!store) return; + try { + await store.settleRun(runId); + } catch (error) { + console.error( + JSON.stringify({ + type: 'start-reservation-settle-failed', + runId, + error: error instanceof Error ? error.message : String(error), + }), + ); + } + } + + /** The fence check every re-entry passes — resume, and the deadline alarm. */ + async #assertResumeFence(runId: string): Promise { + const fence = this.#executionFence; + if (!fence) return; + const reading = await fence.read(); + if (!admitsExistingRun(reading, runId)) { + throw new ExecutionFencedError(reading.state, 'run resume'); + } + } + async start( workflowId: string, options: StartRunOptions, @@ -985,7 +1149,7 @@ export class RunnerRuntime { // (durable-object.ts readJson), and RegExp.test() coerces its argument to a // String — so a numeric runId like 123 would pass the pattern as "123" yet // mint a run keyed by the number 123, unreachable by the string "123" the - // URL path later carries. There is NO generation fallback (INV-1): a + // URL path later carries. There is NO generation fallback: a // missing/null runId is a client error, not a request for one. if (!isPathSafeId(options.runId)) { throw new InvalidRunRequestError( @@ -1019,6 +1183,11 @@ export class RunnerRuntime { ) { throw new InvalidRunRequestError('attemptToken is malformed'); } + // The fence, BEFORE the run lock and before any storage work: a fenced + // deployment must not queue behind a live run's lock just to be refused, + // and must write nothing on the way to the refusal. One read, never + // memoized (execution-fence.ts). + await this.#assertStartFence(runId, options.idempotencyKey); return this.#withRunLock(workflowId, runId, async () => { // Supplied ids can collide with an existing run; starting it // again would re-execute already-executed steps. @@ -1052,7 +1221,7 @@ export class RunnerRuntime { options.storedRequestContext, lifecycle, ); - // Thread the host DO's pubsub identity into the run (CI-M-002-002). Core + // Thread the host DO's pubsub identity into the run. Core // accepts `createRun({ runId, pubsub })` at every one of its OWN call // sites (agent/durable index.js:5224/5541) and stamps it straight onto // `new Run({ ..., pubsub: options?.pubsub })`, defaulting a FRESH @@ -1123,6 +1292,10 @@ export class RunnerRuntime { options: ResumeRunOptions = {}, ): Promise { const workflow = this.#getWorkflow(workflowId); + // A drain must not refuse resumes — the suspended runs it is draining are + // waiting for exactly these — so only migration-locked and proof-only + // block here, and proof-only admits its one nominated run. + await this.#assertResumeFence(runId); return this.#withRunLock(workflowId, runId, async () => { const state = await this.#workflowState(workflow, runId); if (!state) throw new UnknownRunError(workflowId, runId); @@ -1176,7 +1349,7 @@ export class RunnerRuntime { new RequestContext(Object.entries(preparationValues)), ); } - // Same host pubsub identity as start() (CI-M-002-002) — see the note + // Same host pubsub identity as start() — see the note // there; undefined stays byte-identical to `createRun({ runId })`. run = await workflow.createRun({ runId, pubsub: this.#pubsub }); await this.#withLifecycleLock(workflowId, runId, async () => { @@ -1451,6 +1624,10 @@ export class RunnerRuntime { ) { throw new UnknownRunError(workflowId, runId); } + // A re-entry onto an already-terminal run heals a reconcile that an + // earlier crash lost. The CAS is `state <> 'terminal'`, so this is a + // no-op for the reservations that settled the first time. + await this.#settleStartReservation(runId); return { summary: await this.#summaryAfterPersist(workflowId, runId), transitioned: false, @@ -1573,6 +1750,11 @@ export class RunnerRuntime { now, ); this.#terminalAbortIntents.delete(this.#runKey(workflowId, runId)); + // Cancel and timeout are terminal too: a run killed by an operator or by + // its deadline spends its idempotency key exactly as a completed one does, + // and a key left unspent here would keep a dead run in the drain + // inventory forever. + await this.#settleStartReservation(runId); return { summary: await this.#summaryAfterPersist(workflowId, runId), transitioned: true, @@ -2056,6 +2238,11 @@ export class RunnerRuntime { }, }); }); + // The run is terminal and its snapshot now says so, so any idempotency key + // that named it is spent. AFTER the persist, never before: a reservation + // marked terminal ahead of a persist that then failed would answer a retry + // with ALREADY_SETTLED for a run whose settled state exists nowhere. + await this.#settleStartReservation(runId); } #getWorkflow(workflowId: string): AnyWorkflow { diff --git a/packages/flowsafe/src/do-runner/start-idempotency.test.ts b/packages/flowsafe/src/do-runner/start-idempotency.test.ts new file mode 100644 index 0000000..5bb5400 --- /dev/null +++ b/packages/flowsafe/src/do-runner/start-idempotency.test.ts @@ -0,0 +1,1134 @@ +// SPDX-License-Identifier: Apache-2.0 +// The reservation state machine, exercised as real SQL over node:sqlite. +// +// Every assertion here is ultimately about a PAID external call. A workflow's +// first step can wire funds; a second run of it wires them twice. So the cases +// below are written adversarially — two callers racing the same insert, a +// caller probing somebody else's key, a claim taken twice, a purge reaching a +// row whose run is still readable — and each one asserts the EXPENSIVE +// direction: that exactly one caller was told to start. + +import { describe, expect, it } from 'vitest'; + +import { + openSqlite, + type SqliteDatabase, + sqliteUnitDatabase, +} from '../../test-support/sqlite.js'; +import { doErrorResponse } from './do-error-response.js'; +import type { ExecutionFenceDatabase } from './execution-fence.js'; +import { + ExecutionFencedError, + ExecutionFenceStore, +} from './execution-fence.js'; +import { + beginIdempotentStart, + IdempotentStartAlreadySettledError, + IdempotentStartPendingError, + type IdempotentStartSurface, + IdempotentStartUnresolvableError, + InvalidStartIdempotencyRequestError, + isStartReservationRefusal, + requireStartIdempotency, + rollbackFencedStart, + START_IDEMPOTENCY_TABLE, + type StartIdempotencyDatabase, + StartIdempotencyStore, + StartIdempotencyUnsupportedError, + type StartReservation, + StartReservationOwnerMismatchError, + StartReservationTargetMismatchError, + StartReservationUnreadableError, +} from './start-idempotency.js'; + +const OWNER = { kind: 'human', id: 'operator-1' } as const; +const OTHER_OWNER = { kind: 'human', id: 'operator-2' } as const; + +function harness(now: () => number = () => 1_000) { + const sqlite = openSqlite(); + const binding = sqliteUnitDatabase(sqlite) as StartIdempotencyDatabase; + return { + sqlite, + binding, + store: new StartIdempotencyStore(binding, { now }), + }; +} + +function rows(sqlite: SqliteDatabase): Array> { + return sqlite + .prepare(`SELECT * FROM ${START_IDEMPOTENCY_TABLE}`) + .all() as Array>; +} + +function workflowRequest(key: string, runId: string, workflowId = 'payout') { + return { + key, + owner: OWNER, + targetKind: 'workflow' as const, + targetId: workflowId, + mintRunId: () => runId, + }; +} + +/** A surface that has nothing persisted and nothing live — the fresh case. */ +const EMPTY_SURFACE: IdempotentStartSurface = { + persisted: async () => undefined, + live: async () => false, +}; + +describe('start idempotency error taxonomy', () => { + it('publishes the invalid-request status and reason code', async () => { + const error = new InvalidStartIdempotencyRequestError('key is malformed'); + const response = doErrorResponse(error); + + expect(error.status).toBe(400); + expect(error.reason.code).toBe('INVALID_START_IDEMPOTENCY_REQUEST'); + expect(response.status).toBe(400); + expect(await response.json()).toEqual({ + error: 'key is malformed', + reason: { code: 'INVALID_START_IDEMPOTENCY_REQUEST' }, + }); + }); +}); + +describe('isStartReservationRefusal', () => { + it('recognizes every union member and excludes unreadable storage', () => { + const reservation: StartReservation = { + key: 'key-1', + owner: OWNER, + targetKind: 'workflow', + targetId: 'payout', + runId: 'run-1', + state: 'started', + createdAt: 1_000, + updatedAt: 2_000, + }; + const refusals = [ + new StartReservationOwnerMismatchError('key-1'), + new StartReservationTargetMismatchError('key-1', reservation), + new IdempotentStartPendingError(reservation), + new IdempotentStartUnresolvableError(reservation), + new IdempotentStartAlreadySettledError(reservation), + new StartIdempotencyUnsupportedError(), + new InvalidStartIdempotencyRequestError('key is malformed'), + ]; + + expect(refusals.map(isStartReservationRefusal)).toEqual([ + true, + true, + true, + true, + true, + true, + true, + ]); + expect( + isStartReservationRefusal(new StartReservationUnreadableError('key-1')), + ).toBe(false); + }); +}); + +describe('StartIdempotencyStore.reserve', () => { + it('creates the reservation and reports the caller as its creator', async () => { + // #given a key nobody has used + const { store, sqlite } = harness(); + + // #when + const outcome = await store.reserve(workflowRequest('key-1', 'run-1')); + + // #then the caller owns the start, and the row records exactly what it + // minted — the store never generates a run id of its own. + expect(outcome.created).toBe(true); + expect(outcome.reservation).toMatchObject({ + key: 'key-1', + runId: 'run-1', + state: 'reserved', + targetKind: 'workflow', + targetId: 'payout', + owner: { kind: 'human', id: 'operator-1' }, + }); + expect(rows(sqlite)).toHaveLength(1); + }); + + it('gives every later caller the WINNER’s run id, and tells none of them they created it', async () => { + // #given two callers minting DIFFERENT run ids under one key — the shape a + // lost response takes when the client retries into a second isolate + const { store, sqlite } = harness(); + await store.reserve(workflowRequest('key-1', 'run-first')); + + // #when + const second = await store.reserve(workflowRequest('key-1', 'run-second')); + + // #then the loser converges onto the first run rather than starting one: + // two winners is the failure this whole module exists to prevent. + expect(second.created).toBe(false); + expect(second.reservation.runId).toBe('run-first'); + expect(rows(sqlite)).toHaveLength(1); + }); + + it('refuses a key owned by another principal without naming what it holds', async () => { + // #given a key already reserved by someone else + const { store } = harness(); + await store.reserve(workflowRequest('key-1', 'run-1')); + + // #when a different principal probes it + const refusal = await store + .reserve({ + key: 'key-1', + owner: OTHER_OWNER, + targetKind: 'workflow', + targetId: 'payout', + mintRunId: () => 'run-2', + }) + .catch((error: unknown) => error); + + // #then 403, and a reason carrying nothing about the reservation: a key is + // guessable by construction, so this response is reachable by probing. + expect(refusal).toBeInstanceOf(StartReservationOwnerMismatchError); + expect((refusal as StartReservationOwnerMismatchError).status).toBe(403); + expect((refusal as StartReservationOwnerMismatchError).reason).toEqual({ + code: 'IDEMPOTENT_START_OWNER_MISMATCH', + }); + }); + + it('checks the owner BEFORE the target, so a foreign caller learns nothing about either', async () => { + // #given a reservation whose owner AND target both differ from the probe + const { store } = harness(); + await store.reserve(workflowRequest('key-1', 'run-1', 'payout')); + + // #when + const refusal = await store + .reserve({ + key: 'key-1', + owner: OTHER_OWNER, + targetKind: 'workflow', + targetId: 'refund', + mintRunId: () => 'run-2', + }) + .catch((error: unknown) => error); + + // #then owner wins: a target mismatch here would leak 'payout' to a + // principal that has no claim on the key. + expect(refusal).toBeInstanceOf(StartReservationOwnerMismatchError); + }); + + it('refuses the owner’s own key pointed at a different workflow, and names the target it holds', async () => { + // #given + const { store } = harness(); + await store.reserve(workflowRequest('key-1', 'run-1', 'payout')); + + // #when the same principal reuses the key for another workflow + const refusal = await store + .reserve(workflowRequest('key-1', 'run-2', 'refund')) + .catch((error: unknown) => error); + + // #then 409 naming 'payout' — the caller owns this key, so telling it what + // the key means is telling it about its own state. + expect(refusal).toBeInstanceOf(StartReservationTargetMismatchError); + expect((refusal as StartReservationTargetMismatchError).status).toBe(409); + expect((refusal as StartReservationTargetMismatchError).reason).toEqual({ + code: 'IDEMPOTENT_START_TARGET_MISMATCH', + targetKind: 'workflow', + targetId: 'payout', + }); + }); + + it('refuses a key that switches target KIND, not just target id', async () => { + // #given a workflow reservation + const { store } = harness(); + await store.reserve(workflowRequest('key-1', 'run-1', 'payout')); + + // #when the same key names an AGENT called 'payout' + const refusal = await store + .reserve({ + key: 'key-1', + owner: OWNER, + targetKind: 'agent', + targetId: 'payout', + threadId: 'thread-1', + mintRunId: () => 'run-2', + }) + .catch((error: unknown) => error); + + // #then refused: a workflow and an agent that share a name are two + // different execution families and two different charges. + expect(refusal).toBeInstanceOf(StartReservationTargetMismatchError); + }); + + it('requires a thread for an agent reservation and rejects one for a workflow', async () => { + // #given — the thread is the agent run's ADDRESS: without it a retry that + // minted a fresh thread could never reach the original run. + const { store } = harness(); + + // #when / #then + await expect( + store.reserve({ + key: 'key-agent', + owner: OWNER, + targetKind: 'agent', + targetId: 'writer', + mintRunId: () => 'run-1', + }), + ).rejects.toBeInstanceOf(InvalidStartIdempotencyRequestError); + await expect( + store.reserve({ + key: 'key-workflow', + owner: OWNER, + targetKind: 'workflow', + targetId: 'payout', + threadId: 'thread-1', + mintRunId: () => 'run-1', + }), + ).rejects.toBeInstanceOf(InvalidStartIdempotencyRequestError); + }); + + it('rejects a key that is not path-safe', async () => { + // #given — the key is a primary key AND is compared against the execution + // fence's proof key, so an unvalidated one reaches both. + const { store } = harness(); + + // #when / #then + await expect( + store.reserve(workflowRequest('key/../escape', 'run-1')), + ).rejects.toBeInstanceOf(InvalidStartIdempotencyRequestError); + }); + + it('rejects a host mint that is not path-safe rather than storing it', async () => { + // #given a host whose mint returns something the run addressing cannot use + const { store, sqlite } = harness(); + + // #when + await expect( + store.reserve({ + ...workflowRequest('key-1', 'unused'), + mintRunId: () => 'run id with spaces', + }), + ).rejects.toBeInstanceOf(InvalidStartIdempotencyRequestError); + + // #then nothing was written: a stored id the DO name join cannot address + // would be a reservation pointing at an unreachable run. + expect(rows(sqlite)).toHaveLength(0); + }); +}); + +describe('StartIdempotencyStore.claim', () => { + it('lets exactly one of many concurrent callers through', async () => { + // #given one reservation and five callers racing its claim — the + // cross-isolate race the agent surface cannot serialize any other way + const { store } = harness(); + const { reservation } = await store.reserve( + workflowRequest('key-1', 'run-1'), + ); + + // #when + const outcomes = await Promise.all( + Array.from({ length: 5 }, () => + store.claim(reservation.key, reservation.runId), + ), + ); + + // #then exactly one winner. Not "at most one", not "usually one". + expect(outcomes.filter(Boolean)).toHaveLength(1); + }); + + it('refuses a claim naming a different run than the reservation holds', async () => { + // #given + const { store } = harness(); + await store.reserve(workflowRequest('key-1', 'run-1')); + + // #when / #then a claim can never land on a row rewritten underneath it + expect(await store.claim('key-1', 'run-other')).toBe(false); + }); + + it('cannot re-claim a reservation that is already started', async () => { + // #given a claimed reservation + const { store } = harness(); + await store.reserve(workflowRequest('key-1', 'run-1')); + expect(await store.claim('key-1', 'run-1')).toBe(true); + + // #when / #then + expect(await store.claim('key-1', 'run-1')).toBe(false); + }); +}); + +describe('StartIdempotencyStore.release', () => { + it('returns a claim to reserved so a retry after the fence reopens converges', async () => { + // #given a claim taken and then refused by the fence + const { store } = harness(); + await store.reserve(workflowRequest('key-1', 'run-1')); + await store.claim('key-1', 'run-1'); + + // #when + expect(await store.release('key-1', 'run-1')).toBe(true); + + // #then the SAME run id is claimable again — a fence transition mid-start + // must not manufacture an unresolvable reservation out of an operator + // action, nor hand the retry a second run. + expect((await store.read('key-1'))?.state).toBe('reserved'); + expect(await store.claim('key-1', 'run-1')).toBe(true); + expect((await store.read('key-1'))?.runId).toBe('run-1'); + }); + + it('cannot release a reservation that already settled', async () => { + // #given a terminal reservation + const { store } = harness(); + await store.reserve(workflowRequest('key-1', 'run-1')); + await store.claim('key-1', 'run-1'); + await store.settleRun('run-1'); + + // #when / #then a spent key never becomes startable again + expect(await store.release('key-1', 'run-1')).toBe(false); + expect((await store.read('key-1'))?.state).toBe('terminal'); + }); +}); + +describe('StartIdempotencyStore.settleRun', () => { + it('marks the run’s reservation terminal and stamps the horizon from that moment', async () => { + // #given a claimed reservation, and a clock that moves + let now = 1_000; + const { store } = harness(() => now); + await store.reserve(workflowRequest('key-1', 'run-1')); + await store.claim('key-1', 'run-1'); + now = 5_000; + + // #when + expect(await store.settleRun('run-1')).toBe(1); + + // #then + const stored = await store.read('key-1'); + expect(stored?.state).toBe('terminal'); + expect(stored?.updatedAt).toBe(5_000); + }); + + it('is a no-op the second time, so every terminal path may call it', async () => { + // #given — a run can reach terminal by completing, failing, being cancelled + // or timing out, and those paths do not coordinate. + const { store } = harness(); + await store.reserve(workflowRequest('key-1', 'run-1')); + await store.settleRun('run-1'); + + // #when / #then + expect(await store.settleRun('run-1')).toBe(0); + }); + + it('settles nothing for a run nobody reserved', async () => { + // #given the overwhelmingly common case: a run started without a key + const { store } = harness(); + await store.reserve(workflowRequest('key-1', 'run-1')); + + // #when / #then + expect(await store.settleRun('run-unrelated')).toBe(0); + }); +}); + +describe('StartIdempotencyStore against a missing table', () => { + it('reads as absent, and neither claims nor settles', async () => { + // #given a database on which no key has ever been used, so the lazy DDL + // has never run + const { store } = harness(); + + // #when / #then absence is not a fault — it is an empty table by another + // name — but it must also never look like a successful transition. + expect(await store.read('key-1')).toBeUndefined(); + expect(await store.claim('key-1', 'run-1')).toBe(false); + expect(await store.release('key-1', 'run-1')).toBe(false); + expect(await store.settleRun('run-1')).toBe(0); + expect(await store.reservationsForRuns(['run-1'])).toEqual([]); + }); + + it('creates NOTHING on a read — the inventory sweep must not be a write', async () => { + // #given + const { store, sqlite } = harness(); + + // #when + await store.read('key-1'); + await store.reservationsForRuns(['run-1']); + + // #then no lazy DDL: a read path that emits CREATE TABLE is a write path + // wearing a read's name, and the drain inventory reads this table on every + // sweep of a deployment that is deliberately not executing. + const tables = sqlite + .prepare(`SELECT name FROM sqlite_master WHERE type='table' AND name = ?`) + .all(START_IDEMPOTENCY_TABLE); + expect(tables).toEqual([]); + }); + + it('degrades CLOSED when the store cannot be read for any other reason', async () => { + // #given a binding whose reads fail with something that is NOT a missing + // table — the case where answering "no reservation" would start a run + const failing: StartIdempotencyDatabase = { + prepare: () => ({ + bind: () => failing.prepare('') as never, + run: async () => { + throw new Error('D1_ERROR: network'); + }, + all: async () => { + throw new Error('D1_ERROR: network'); + }, + }), + }; + const store = new StartIdempotencyStore(failing); + + // #when / #then a 503, never `undefined` + await expect(store.read('key-1')).rejects.toMatchObject({ + status: 503, + reason: { code: 'IDEMPOTENT_START_UNREADABLE' }, + }); + }); +}); + +describe('beginIdempotentStart', () => { + it('tells the first caller to start and every retry to replay the same run', async () => { + // #given a run that persisted after the first start + const { store } = harness(); + const persisted = new Map(); + const surface: IdempotentStartSurface = { + persisted: async (reservation) => persisted.get(reservation.runId), + live: async () => false, + }; + + // #when the first call starts, records a summary, and a retry arrives + const first = await beginIdempotentStart( + store, + workflowRequest('key-1', 'run-1'), + surface, + ); + expect(first.kind).toBe('start'); + persisted.set('run-1', 'summary-of-run-1'); + const retry = await beginIdempotentStart( + store, + workflowRequest('key-1', 'run-2'), + surface, + ); + + // #then the retry answers with the FIRST run's state and starts nothing + expect(retry).toEqual({ + kind: 'replay', + reservation: expect.objectContaining({ runId: 'run-1' }), + persisted: 'summary-of-run-1', + }); + }); + + it('answers with the persisted run even when the reservation still reads started', async () => { + // #given a reservation whose terminal reconcile was lost to a crash, but + // whose run persisted + const { store } = harness(); + await store.reserve(workflowRequest('key-1', 'run-1')); + await store.claim('key-1', 'run-1'); + + // #when + const decision = await beginIdempotentStart( + store, + workflowRequest('key-1', 'run-2'), + { persisted: async () => 'summary', live: async () => false }, + ); + + // #then the persisted state wins over the row's state: a stale `started` + // must not refuse a retry whose run is sitting right there, finished. + expect(decision.kind).toBe('replay'); + }); + + it('lets a retry re-claim a reservation whose first caller died BEFORE the claim', async () => { + // #given a bare reservation — the crash window between insert and claim, + // in which nothing has executed + const { store } = harness(); + await store.reserve(workflowRequest('key-1', 'run-1')); + + // #when + const decision = await beginIdempotentStart( + store, + workflowRequest('key-1', 'run-ignored'), + EMPTY_SURFACE, + ); + + // #then it proceeds with the RESERVED run id, not a fresh one: converging + // here is what makes a crashed reservation self-healing instead of a key + // that can never be used again. + expect(decision).toMatchObject({ + kind: 'start', + reservation: { runId: 'run-1', state: 'reserved' }, + }); + }); + + it('refuses a claimed-but-unpersisted run as PENDING while its host is executing it', async () => { + // #given a claim held by a run that is genuinely still working — the + // normal in-flight window, which is legitimately unbounded because the + // first persisted summary lands only at the first suspend or terminal + let now = 1_000; + const { store } = harness(() => now); + await store.reserve(workflowRequest('key-1', 'run-1')); + now = 2_500; + await store.claim('key-1', 'run-1'); + + // #when + const refusal = await beginIdempotentStart( + store, + workflowRequest('key-1', 'run-2'), + { persisted: async () => undefined, live: async () => true }, + ).catch((error: unknown) => error); + + // #then 503 with the claim's own timestamp — retryable, and no timer + // anywhere: a bound on legitimate work would misclassify a long live run + // and invite a fresh key and a second charge. + expect(refusal).toBeInstanceOf(IdempotentStartPendingError); + expect((refusal as IdempotentStartPendingError).status).toBe(503); + expect((refusal as IdempotentStartPendingError).reason).toEqual({ + code: 'IDEMPOTENT_START_PENDING', + runId: 'run-1', + pendingSince: 2_500, + }); + }); + + it('refuses a claimed-but-unpersisted run as UNRESOLVABLE when nothing is running it', async () => { + // #given the one genuinely ambiguous state: the claim was taken, nothing + // persisted, and the host that took it is gone + const { store } = harness(); + await store.reserve(workflowRequest('key-1', 'run-1')); + await store.claim('key-1', 'run-1'); + + // #when + const refusal = await beginIdempotentStart( + store, + workflowRequest('key-1', 'run-2'), + EMPTY_SURFACE, + ).catch((error: unknown) => error); + + // #then 409 and NEVER a re-execution: whether the first step already took + // effect is unknowable here, and the message says so. + expect(refusal).toBeInstanceOf(IdempotentStartUnresolvableError); + expect((refusal as IdempotentStartUnresolvableError).status).toBe(409); + expect((refusal as IdempotentStartUnresolvableError).reason).toEqual({ + code: 'IDEMPOTENT_START_UNRESOLVABLE', + runId: 'run-1', + }); + expect((refusal as Error).message).toMatch(/fresh key/); + }); + + it('refuses a settled key whose summary has aged out as ALREADY_SETTLED', async () => { + // #given a completed run whose snapshot the retention purge removed + const { store } = harness(); + await store.reserve(workflowRequest('key-1', 'run-1')); + await store.claim('key-1', 'run-1'); + await store.settleRun('run-1'); + + // #when + const refusal = await beginIdempotentStart( + store, + workflowRequest('key-1', 'run-2'), + EMPTY_SURFACE, + ).catch((error: unknown) => error); + + // #then the work is done even though nobody can still read the outcome. + // The reservation outliving the snapshot is the ONLY reason this answer + // exists rather than a fresh key and a second run. + expect(refusal).toBeInstanceOf(IdempotentStartAlreadySettledError); + expect((refusal as IdempotentStartAlreadySettledError).reason).toEqual({ + code: 'IDEMPOTENT_START_ALREADY_SETTLED', + runId: 'run-1', + }); + }); + + it('never returns `start` twice for one key, however many callers race it', async () => { + // #given ten concurrent first-calls on one key against one database — the + // shape of a client retrying into parallel isolates + const { store } = harness(); + let mints = 0; + const surface: IdempotentStartSurface = { + persisted: async () => undefined, + live: async () => false, + }; + + // #when + const decisions = await Promise.allSettled( + Array.from({ length: 10 }, () => + beginIdempotentStart( + store, + { + key: 'key-1', + owner: OWNER, + targetKind: 'workflow', + targetId: 'payout', + mintRunId: () => { + mints += 1; + return `run-${mints}`; + }, + }, + surface, + ), + ), + ); + + // #then exactly one caller was told to start, and every other was refused + // or told to replay. Two `start` decisions is two executions. + const starts = decisions.filter( + (outcome) => + outcome.status === 'fulfilled' && outcome.value.kind === 'start', + ); + expect(starts).toHaveLength(1); + }); + + it('lets the CREATOR lose the claim to a later caller, and refuses the creator rather than starting it', async () => { + // #given the interleaving the two-signal `created` test cannot reach by + // racing: caller A wins the INSERT and caller B wins the CAS. Nothing about + // creating the row entitles A to start it, and if both ever believed they + // could, the key would have bought nothing. + const { store } = harness(); + const live: IdempotentStartSurface = { + persisted: async () => undefined, + // The winner IS executing — the realistic state of the world at the + // moment the loser asks. + live: async () => true, + }; + const decisions: string[] = []; + let loserDecision: unknown; + const realClaim = store.claim.bind(store); + let interleaved = false; + store.claim = async (key: string, runId: string) => { + if (!interleaved) { + // B arrives in the window between A's insert and A's claim. + interleaved = true; + loserDecision = await beginIdempotentStart( + store, + workflowRequest('key-1', 'run-B'), + live, + ); + } + return realClaim(key, runId); + }; + + // #when A (the creator) races its own claim against B's + const refusal = await beginIdempotentStart( + store, + workflowRequest('key-1', 'run-A'), + live, + ).catch((error: unknown) => error); + for (const decision of [loserDecision, refusal]) { + if ( + decision !== null && + typeof decision === 'object' && + 'kind' in decision + ) { + decisions.push(String((decision as { kind: string }).kind)); + } + } + + // #then exactly one caller was told to start, it was B, and the run it + // starts is the one A RESERVED — the reservation decides the run id, the + // claim only decides who executes it. + expect(decisions).toEqual(['start']); + expect(loserDecision).toMatchObject({ + kind: 'start', + reservation: { runId: 'run-A' }, + }); + // And the creator is REFUSED, pointed at the run its own key already names. + expect(refusal).toBeInstanceOf(IdempotentStartPendingError); + expect((refusal as IdempotentStartPendingError).reason.runId).toBe('run-A'); + }); + + it('re-claims the reserved run after a crash between reserve and claim, and stores no second id', async () => { + // #given a reservation whose creator died before claiming — nothing has + // executed, and the row is the only thing that knows which run this key + // means + const { store, sqlite } = harness(); + await store.reserve(workflowRequest('key-1', 'run-1')); + let minted = 0; + + // #when the retry arrives, mints its own candidate, and is resolved + const decision = await beginIdempotentStart( + store, + { + key: 'key-1', + owner: OWNER, + targetKind: 'workflow', + targetId: 'payout', + mintRunId: () => { + minted += 1; + return 'run-retry-candidate'; + }, + }, + EMPTY_SURFACE, + ); + + // #then it starts the FIRST run, and the candidate it minted is discarded + // rather than stored: a second id in this table is a second run waiting to + // happen, whatever the decision said. + expect(decision).toMatchObject({ + kind: 'start', + reservation: { runId: 'run-1' }, + }); + expect(minted).toBe(1); + expect(rows(sqlite)).toEqual([ + expect.objectContaining({ key: 'key-1', run_id: 'run-1' }), + ]); + }); + + it('refuses a key belonging to another SURFACE’s principal, in both directions, and says nothing else', async () => { + // #given the collision a shared key namespace makes reachable: a public + // human actor on the run router and a stamped agent principal on the agent + // topology, deriving the same key from the same order id + const { store } = harness(); + const humanOwner = { kind: 'human', id: 'opal' } as const; + const agentOwner = { kind: 'agent', id: 'writer-agent' } as const; + const agentRequest = (key: string) => ({ + key, + owner: agentOwner, + targetKind: 'agent' as const, + targetId: 'writer', + threadId: 'thread-1', + mintRunId: () => 'agent-run', + }); + const humanRequest = (key: string) => ({ + key, + owner: humanOwner, + targetKind: 'workflow' as const, + targetId: 'payout', + mintRunId: () => 'workflow-run', + }); + + // #when each surface reserves first and the other follows + await store.reserve(humanRequest('key-human-first')); + const agentRefusal = await store + .reserve(agentRequest('key-human-first')) + .catch((error: unknown) => error); + await store.reserve(agentRequest('key-agent-first')); + const humanRefusal = await store + .reserve(humanRequest('key-agent-first')) + .catch((error: unknown) => error); + + // #then both directions refuse with 403 and an EMPTY body — no target, no + // kind, no run. A key is guessable by construction, so this refusal is + // reachable by probing, and anything it named would be something the prober + // learned about a principal it has no claim on. + for (const refusal of [agentRefusal, humanRefusal]) { + expect(refusal).toBeInstanceOf(StartReservationOwnerMismatchError); + expect((refusal as StartReservationOwnerMismatchError).status).toBe(403); + expect((refusal as StartReservationOwnerMismatchError).reason).toEqual({ + code: 'IDEMPOTENT_START_OWNER_MISMATCH', + }); + } + }); + + it('refuses rather than starting when the reservation row cannot be read', async () => { + // #given a row this build cannot parse — here a corrupt `updated_at`, the + // column the purge horizon and `pendingSince` are both measured from + const { store, sqlite } = harness(); + await store.reserve(workflowRequest('key-1', 'run-1')); + sqlite + .prepare( + `UPDATE ${START_IDEMPOTENCY_TABLE} SET updated_at = 'corrupt' WHERE key = ?`, + ) + .run('key-1'); + + // #when the key is consulted again + const refusal = await beginIdempotentStart( + store, + workflowRequest('key-1', 'run-2'), + EMPTY_SURFACE, + ).catch((error: unknown) => error); + + // #then 503, and never the absent answer. "There is no reservation" is the + // answer that STARTS A RUN, so it must be unreachable from a row that + // cannot be understood — a corrupt terminal `updated_at` would otherwise + // read as epoch 0 and make the row immediately reapable as well. + expect(refusal).toBeInstanceOf(StartReservationUnreadableError); + expect((refusal as StartReservationUnreadableError).status).toBe(503); + expect((refusal as StartReservationUnreadableError).reason).toEqual({ + code: 'IDEMPOTENT_START_UNREADABLE', + }); + }); +}); + +describe('rollbackFencedStart', () => { + it('gives the claim back for a fence refusal and re-throws it unchanged', async () => { + // #given a claim consumed by a start the fence refused — provably + // pre-execution, because the fence is read before the run lock + const { store } = harness(); + await store.reserve(workflowRequest('key-1', 'run-1')); + await store.claim('key-1', 'run-1'); + const fenced = new ExecutionFencedError('migration-locked', 'run start'); + + // #when + const thrown = await rollbackFencedStart( + store, + 'key-1', + 'run-1', + fenced, + ).catch((error: unknown) => error); + + // #then the caller still sees the fence's own refusal, and the key is + // usable again once the operator reopens. + expect(thrown).toBe(fenced); + expect((await store.read('key-1'))?.state).toBe('reserved'); + }); + + it('recognizes a fence refusal that crossed a Durable Object boundary', async () => { + // #given the shape a fenced run-DO start takes on the Worker side: the + // class is gone, the status and structured reason survive + const { store } = harness(); + await store.reserve(workflowRequest('key-1', 'run-1')); + await store.claim('key-1', 'run-1'); + const wire = Object.assign(new Error('deployment execution is fenced'), { + status: 503, + reason: { code: 'EXECUTION_FENCED', state: 'migration-locked' }, + }); + + // #when + await rollbackFencedStart(store, 'key-1', 'run-1', wire).catch( + () => undefined, + ); + + // #then rolled back all the same — an instanceof-only test would answer + // "not a fence refusal" for every caller on the far side of the boundary, + // which is where the run router actually sits. + expect((await store.read('key-1'))?.state).toBe('reserved'); + }); + + it('KEEPS the claim for any other start failure', async () => { + // #given a start that failed for a reason that may well have executed + const { store } = harness(); + await store.reserve(workflowRequest('key-1', 'run-1')); + await store.claim('key-1', 'run-1'); + + // #when + await rollbackFencedStart( + store, + 'key-1', + 'run-1', + new Error('step exploded'), + ).catch(() => undefined); + + // #then still claimed: releasing here would hand the next retry a second + // run after a start that may already have charged somebody. + expect((await store.read('key-1'))?.state).toBe('started'); + }); + + it('completes the whole round trip: claim, fence refusal, release, reopen, and ONE execution of the same run', async () => { + // #given a real fence and a real reservation over one database, and a host + // whose start executes paid work — the shape the round trip has to be + // proved in, because each half of it is only correct given the other. + const sqlite = openSqlite(); + const binding = sqliteUnitDatabase(sqlite); + const store = new StartIdempotencyStore( + binding as StartIdempotencyDatabase, + ); + const fence = new ExecutionFenceStore(binding as ExecutionFenceDatabase); + await fence.seed('open'); + await fence.transition({ expected: 'open', next: 'migration-locked' }); + let executions = 0; + const startRun = async (): Promise => { + const reading = await fence.read(); + if (reading.state !== 'open') { + throw new ExecutionFencedError(reading.state, 'run start'); + } + executions += 1; + }; + const attempt = async (candidate: string): Promise => { + const decision = await beginIdempotentStart( + store, + workflowRequest('key-1', candidate), + EMPTY_SURFACE, + ); + if (decision.kind !== 'start') throw new Error('expected a start'); + const { key, runId } = decision.reservation; + try { + await startRun(); + } catch (error) { + return rollbackFencedStart(store, key, runId, error); + } + return decision.reservation; + }; + + // #when the fenced attempt is refused, the operator reopens, and the client + // retries with the same key + const refused = await attempt('run-1').catch((error: unknown) => error); + expect(refused).toBeInstanceOf(ExecutionFencedError); + expect((await store.read('key-1'))?.state).toBe('reserved'); + await fence.transition({ expected: 'migration-locked', next: 'open' }); + const started = await attempt('run-ignored'); + + // #then the retry ran the SAME run the fenced attempt reserved, exactly + // once. A rollback that did not land would have left the key UNRESOLVABLE + // forever; a rollback that handed back a fresh run id would have made an + // operator's drain the cause of a second charge. + expect(started.runId).toBe('run-1'); + expect(executions).toBe(1); + expect((await store.read('key-1'))?.state).toBe('started'); + }); + + it('still re-throws the fence refusal when the release itself fails', async () => { + // #given a claimed reservation and a store whose release cannot be written + // — a storage incident arriving during a deployment that is already + // refusing to execute + const { store } = harness(); + await store.reserve(workflowRequest('key-1', 'run-1')); + await store.claim('key-1', 'run-1'); + store.release = async () => { + throw new Error('D1_ERROR: network'); + }; + const fenced = new ExecutionFencedError('migration-locked', 'run start'); + + // #when + const thrown = await rollbackFencedStart( + store, + 'key-1', + 'run-1', + fenced, + ).catch((error: unknown) => error); + + // #then the caller sees the FENCE's refusal, not the storage error: + // swallowing it would leave the caller believing the deployment is broken + // rather than fenced, and the rollback's own failure is not something the + // caller can act on. + expect(thrown).toBe(fenced); + // The claim stayed taken, so the key is now UNRESOLVABLE rather than + // startable — recoverable by investigation, which is the direction this + // best-effort rollback deliberately fails in. + expect((await store.read('key-1'))?.state).toBe('started'); + await expect( + beginIdempotentStart( + store, + workflowRequest('key-1', 'run-2'), + EMPTY_SURFACE, + ), + ).rejects.toBeInstanceOf(IdempotentStartUnresolvableError); + }); +}); + +describe('requireStartIdempotency', () => { + it('refuses a key on a host that wired no store', () => { + // #given / #when / #then honouring the key silently would answer an + // exactly-once REQUEST with at-least-once BEHAVIOUR, and the caller would + // have no way to find out. + expect(() => requireStartIdempotency('none')).toThrow( + StartIdempotencyUnsupportedError, + ); + expect(() => requireStartIdempotency(undefined)).toThrow( + StartIdempotencyUnsupportedError, + ); + }); + + it('returns the store when one is wired', () => { + // #given + const { store } = harness(); + + // #then + expect(requireStartIdempotency(store)).toBe(store); + }); +}); + +describe('reservationsForRuns', () => { + it('returns every reservation naming the given runs', async () => { + // #given — the drain inventory and the purge both ask this question + const { store } = harness(); + await store.reserve(workflowRequest('key-1', 'run-1')); + await store.reserve(workflowRequest('key-2', 'run-2')); + + // #when + const found = await store.reservationsForRuns(['run-2', 'run-missing']); + + // #then + expect( + found.map((reservation: StartReservation) => reservation.key), + ).toEqual(['key-2']); + }); +}); + +describe('proof-only composition', () => { + it('re-asserts the proof binding on a REPLAY, so a run whose binding was lost stays resumable', async () => { + // #given a proof-only deployment whose proof run already exists and + // persisted, but whose fence has lost its proof_run_id — the shape left by + // a fence moved away and back onto the same key while the run survived. + // Without the binding, proof-only admits no resume for it at all. + const sqlite = openSqlite(); + const binding = sqliteUnitDatabase(sqlite); + const store = new StartIdempotencyStore( + binding as StartIdempotencyDatabase, + ); + const fence = new ExecutionFenceStore(binding as ExecutionFenceDatabase); + await fence.seed('migration-locked'); + await fence.transition({ + expected: 'migration-locked', + next: 'proof-only', + proofKey: 'proof-key-1', + }); + await store.reserve({ + ...workflowRequest('proof-key-1', 'proof-run'), + key: 'proof-key-1', + }); + await store.claim('proof-key-1', 'proof-run'); + expect((await fence.read()).proofRunId).toBeUndefined(); + + // #when a retry carrying the same key finds the run persisted + const decision = await beginIdempotentStart( + store, + workflowRequest('proof-key-1', 'ignored'), + { persisted: async () => 'summary', live: async () => false }, + fence, + ); + + // #then the replay answered with the run's state AND put the binding back + expect(decision.kind).toBe('replay'); + await expect(fence.read()).resolves.toEqual({ + state: 'proof-only', + proofKey: 'proof-key-1', + proofRunId: 'proof-run', + }); + }); + + it('changes nothing on a replay whose key is not the nominated proof key', async () => { + // #given a proof-only fence nominating a DIFFERENT key + const sqlite = openSqlite(); + const binding = sqliteUnitDatabase(sqlite); + const store = new StartIdempotencyStore( + binding as StartIdempotencyDatabase, + ); + const fence = new ExecutionFenceStore(binding as ExecutionFenceDatabase); + await fence.seed('migration-locked'); + await fence.transition({ + expected: 'migration-locked', + next: 'proof-only', + proofKey: 'proof-key-1', + }); + await store.reserve(workflowRequest('other-key', 'other-run')); + await store.claim('other-key', 'other-run'); + + // #when + await beginIdempotentStart( + store, + workflowRequest('other-key', 'ignored'), + { persisted: async () => 'summary', live: async () => false }, + fence, + ); + + // #then the proof slot is untouched: every guard lives in recordProofRun's + // own CAS, so an unrelated key is zero rows and no harm. + await expect(fence.read()).resolves.toEqual({ + state: 'proof-only', + proofKey: 'proof-key-1', + }); + }); + + it('does not fail a replay when the proof re-bind cannot be written', async () => { + // #given a fence whose write-back throws — a storage incident during a + // replay of a run that already happened + const { store } = harness(); + await store.reserve(workflowRequest('key-1', 'run-1')); + await store.claim('key-1', 'run-1'); + const failing = { + recordProofRun: async () => { + throw new Error('D1_ERROR: network'); + }, + } as unknown as ExecutionFenceStore; + + // #when / #then the caller still gets the run's persisted state: refusing + // the read would answer a successful retry with an error while changing + // nothing about the run. + const decision = await beginIdempotentStart( + store, + workflowRequest('key-1', 'run-2'), + { persisted: async () => 'summary', live: async () => false }, + failing, + ); + expect(decision.kind).toBe('replay'); + }); +}); diff --git a/packages/flowsafe/src/do-runner/start-idempotency.ts b/packages/flowsafe/src/do-runner/start-idempotency.ts new file mode 100644 index 0000000..e5a5231 --- /dev/null +++ b/packages/flowsafe/src/do-runner/start-idempotency.ts @@ -0,0 +1,1281 @@ +// SPDX-License-Identifier: Apache-2.0 +// Owner-bound idempotent start — the reservation that makes "start this once" +// mean once, across retries, isolates, and deployments. +// +// WHY it exists: a run start is the moment a deployment commits to spending +// money. The first step of a workflow can wire funds, file an order, or call a +// paid API, and every layer between a caller and that step is allowed to lose a +// RESPONSE without losing the WORK — a Worker eviction, a client timeout, a +// load balancer retry, an operator re-running a script. Without a reservation +// the only honest answer to "did my start land?" is "retry and find out", and +// that answer charges the card twice. +// +// The reservation is what a key BUYS: a durable row, written before anything +// executes, that says which run this key already means. A retry carrying the +// same key does not start a second run — it finds the first one and is told +// what happened to it. +// +// THE TAXONOMY IS THE CONTRACT. Eight structured reason codes say what flowsafe +// KNOWS, because the caller's next action differs for each and a collapsed code +// would make them all "retry". Five are reservation-decision refusals: +// +// IDEMPOTENT_START_OWNER_MISMATCH (403) this key is somebody else's. Keys are +// owner-scoped, so one tenant principal +// cannot probe, hijack, or collide with +// another's — and never learns more than +// "not yours". +// IDEMPOTENT_START_TARGET_MISMATCH (409) the key is yours but names a +// different workflow/agent than last time. +// Reusing a key across targets is a caller +// bug, and silently honouring it would make +// one key mean two different charges. +// IDEMPOTENT_START_PENDING (503) the run this key names is RUNNING right +// now. Retryable, and legitimately unbounded: +// the first persisted summary lands at the +// first suspend or terminal state, so a long +// live run has no summary yet and is not lost. +// `pendingSince` lets a caller reason about +// how long, without this package pretending a +// timeout would be safe. +// IDEMPOTENT_START_UNRESOLVABLE (409) the claim was taken, nothing persisted, +// and nothing is running. Whether a side +// effect fired before the crash is UNKNOWABLE +// to flowsafe. So it refuses, and says so, and +// never re-executes on its own. A host that +// investigates and decides to re-run does it +// with a FRESH key — a deliberate second +// charge, not one this package invented. +// IDEMPOTENT_START_ALREADY_SETTLED (409) the run finished and its summary has +// aged out. The reservation deliberately +// OUTLIVES the snapshot so this answer exists +// at all; the alternative is a purged run +// looking exactly like a fresh key. +// +// Three complete the public keyed-start taxonomy: +// +// IDEMPOTENT_START_UNSUPPORTED (503) a keyed start reached a deployment whose +// host did not wire a reservation store. +// INVALID_START_IDEMPOTENCY_REQUEST (400) the keyed input was malformed. +// IDEMPOTENT_START_UNREADABLE (503) the reservation store could not be read +// or contained a row this build cannot parse. +// +// The seven-member `StartReservationRefusal` union recognized by +// `isStartReservationRefusal` includes everything above except UNREADABLE, +// which propagates separately as an integrity or availability failure. +// +// NO TIMER ANYWHERE. Two of those branches are separated by a LIVENESS PROBE, +// never by elapsed time. A timer would have to guess a bound on legitimate +// in-flight work, and every guess is wrong in the expensive direction: too +// short and a long live run is declared dead, inviting a fresh key and a second +// charge; too long and a genuinely crashed start wedges its key. The probe asks +// the run's own host whether it is executing, which is the question a timer was +// only ever approximating. +// +// THE CLAIM IS THE SERIALIZER. `reserve()` decides which runId a key means; +// `claim()` decides who gets to START it. The claim is one conditional UPDATE, +// so exactly one caller changes a row and every other caller reads the outcome +// instead of racing it. That matters most where Durable Object serialization +// cannot help: agent runs live in thread objects keyed by threadId, so two +// same-key starts naming different threads are two different objects with no +// shared lock at all. This CAS is the only thing between them. +// +// RUN IDS ARE NEVER MINTED HERE. Run ids are server-minted, at the host's own +// existing mint sites; the reservation STORES one and hands the same id back to +// every later caller. A store that generated ids would be a second minting +// authority, and the whole rule is that there is exactly one. + +import { + EXECUTION_PRINCIPAL_KINDS, + type ExecutionPrincipalKind, + isExecutionPrincipalId, + isExecutionPrincipalKind, +} from '../approval-api/principal-identity.js'; +import { missingTableReadsEmpty } from './cause-chain.js'; +import { DoStatusError } from './do-status-error.js'; +import type { ExecutionFenceWiring } from './execution-fence.js'; +import { isExecutionFenceRefusal } from './execution-fence.js'; +import { isPathSafeId } from './path-safe-id.js'; + +/** + * The reservation table — flowsafe-owned, so outside the `mastra_%` schema + * guard, and created lazily by the first `reserve()` rather than by the + * provisioning protocol. Unlike the execution fence (whose ABSENCE has to read + * as a state, so provisioning writes an explicit row), an absent reservation + * table simply means no key has ever been used on this deployment, which is + * indistinguishable from an empty one. + */ +export const START_IDEMPOTENCY_TABLE = 'flowsafe_start_idempotency'; + +export const START_RESERVATION_STATES = [ + 'reserved', + 'started', + 'terminal', +] as const; + +/** + * Where a reservation is in its life: + * + * reserved the key means this runId, and nobody has started it yet + * started one caller won the claim and is (or was) executing + * terminal the run reached a terminal state; the key is spent + * + * The states only ever move forward, with ONE exception: a start refused by the + * execution fence rolls `started` back to `reserved` (see `release`), because a + * fence refusal is the one failure that provably executed nothing. + */ +export type StartReservationState = (typeof START_RESERVATION_STATES)[number]; + +export const START_TARGET_KINDS = ['workflow', 'agent'] as const; + +/** Which execution family a key names — a workflow run, or an agent run. */ +export type StartTargetKind = (typeof START_TARGET_KINDS)[number]; + +/** + * WHO a key belongs to. An execution principal, projected to the same two + * fields `ResourceOwner` carries, and for the same reason: a key is a + * capability to converge on somebody's run, so it must be scoped to whoever + * created it and unforgeable from tenant traffic. + */ +export interface StartReservationOwner { + readonly kind: ExecutionPrincipalKind; + readonly id: string; +} + +/** One reservation row, as every surface reads it. */ +export interface StartReservation { + readonly key: string; + readonly owner: StartReservationOwner; + readonly targetKind: StartTargetKind; + readonly targetId: string; + readonly runId: string; + /** + * The agent run's thread, when the target is an agent. It is the run's + * ADDRESS: a workflow run is reachable from (workflowId, runId) alone, but an + * agent run lives in a thread object and a retry that minted a fresh thread + * would otherwise have no way back to the original. Absent for workflows, + * where storing a derivable address would be a second source of truth. + */ + readonly threadId?: string; + readonly state: StartReservationState; + /** + * Epoch ms of the reserve that created this row. Provenance only: the purge + * horizon is measured from `updatedAt`, so that a key's validity runs from + * the moment it was SPENT rather than from the moment it was first used — + * a long run must not age its own reservation out while it is still running. + */ + readonly createdAt: number; + /** + * Epoch ms of the last state change — `pendingSince` on a live claim, and the + * column the purge horizon is measured from once the row is terminal. + */ + readonly updatedAt: number; +} + +export interface StartReservationRequest { + /** + * `unknown` rather than `string`, the same posture (and for the same reason) + * as `ExecutionFenceTransition.proofKey`: every caller is a route holding a + * parsed JSON body, `assertKey` already validates this against + * PATH_SAFE_ID_PATTERN and throws on anything else, and typing it `string` + * only made callers write `body.idempotencyKey as string` — an assertion that + * is false exactly when the caller sent the wrong thing, so the one input + * this field exists to police would arrive pre-blessed at the type level. + */ + key: unknown; + owner: StartReservationOwner; + targetKind: StartTargetKind; + targetId: string; + /** + * The host's own run-id mint — `context.newRunId()` on the run router, the + * thread topology's minted id on the agent surface. Run ids are server-minted + * and this store never generates one; it only ever stores what it is handed. + * + * A THUNK rather than a value so the store, not the caller, decides when an + * id is needed: the caller that loses the insert never uses its candidate, + * and a host whose mint is expensive or audited should not pay for one it + * throws away. (The mints in this package are pure `crypto.randomUUID()` + * derivations, so a discarded candidate costs nothing and leaks nothing.) + */ + mintRunId: () => string; + /** Required for `targetKind: 'agent'`, rejected for 'workflow'. */ + threadId?: string; +} + +export interface StartReservationOutcome { + /** The authoritative reservation — this caller's, or the winner's. */ + reservation: StartReservation; + /** + * Whether THIS call created the row. Only a creator may go straight to the + * claim; everyone else takes the replay path, which is where the "what + * happened to the first one?" answers live. + */ + created: boolean; +} + +/** + * The key names a reservation owned by a different principal. + * + * 403 rather than 404: the caller sent a syntactically valid key it is simply + * not entitled to, and this is the one refusal whose body deliberately carries + * nothing else — not the owner, not the target, not the run. A key is guessable + * by construction (hosts derive them from order ids and request ids), so this + * response is reachable by probing, and everything it does not say is something + * a prober does not learn. + */ +export class StartReservationOwnerMismatchError extends DoStatusError { + readonly status = 403; + readonly reason: { readonly code: 'IDEMPOTENT_START_OWNER_MISMATCH' }; + + constructor(key: string) { + super(`idempotency key '${key}' belongs to another principal`); + this.name = 'StartReservationOwnerMismatchError'; + this.reason = { code: 'IDEMPOTENT_START_OWNER_MISMATCH' }; + } +} + +/** + * The caller's own key, pointed at a different workflow or agent than the one + * it reserved. 409, and it DOES name the reservation's target: the caller owns + * this key, so telling it what the key already means is telling it about its + * own state, and a caller that reused a key by accident needs exactly that to + * find the bug. + */ +export class StartReservationTargetMismatchError extends DoStatusError { + readonly status = 409; + readonly reason: { + readonly code: 'IDEMPOTENT_START_TARGET_MISMATCH'; + readonly targetKind: StartTargetKind; + readonly targetId: string; + }; + + constructor(key: string, reservation: StartReservation) { + super( + `idempotency key '${key}' already names ${reservation.targetKind} '${reservation.targetId}'`, + ); + this.name = 'StartReservationTargetMismatchError'; + this.reason = { + code: 'IDEMPOTENT_START_TARGET_MISMATCH', + targetKind: reservation.targetKind, + targetId: reservation.targetId, + }; + } +} + +/** + * The key's run is executing right now. 503 for the same reason every + * operator-transient refusal in this package is: the condition is real, it is + * nobody's mistake, and it clears on its own — so a client that honours + * retry semantics converges instead of giving up. + */ +export class IdempotentStartPendingError extends DoStatusError { + readonly status = 503; + readonly reason: { + readonly code: 'IDEMPOTENT_START_PENDING'; + readonly runId: string; + /** Epoch ms of the claim, so a caller can reason about how long. */ + readonly pendingSince: number; + }; + + constructor(reservation: StartReservation) { + super( + `run '${reservation.runId}' for this idempotency key is still starting`, + ); + this.name = 'IdempotentStartPendingError'; + this.reason = { + code: 'IDEMPOTENT_START_PENDING', + runId: reservation.runId, + pendingSince: reservation.updatedAt, + }; + } +} + +/** + * The claim was taken, nothing persisted, and nothing is running. + * + * This is the refusal that refuses to guess. The message is written for the + * human who will read it in a log at 3am, because the decision it asks for is + * a judgement call this package cannot make: a side effect may or may not have + * fired before the crash, and only the host knows whether its first step is the + * kind that charges. + */ +export class IdempotentStartUnresolvableError extends DoStatusError { + readonly status = 409; + readonly reason: { + readonly code: 'IDEMPOTENT_START_UNRESOLVABLE'; + readonly runId: string; + }; + + constructor(reservation: StartReservation) { + super( + `run '${reservation.runId}' for this idempotency key was claimed but never persisted and is not running — investigate whether its first step already took effect before re-running with a fresh key`, + ); + this.name = 'IdempotentStartUnresolvableError'; + this.reason = { + code: 'IDEMPOTENT_START_UNRESOLVABLE', + runId: reservation.runId, + }; + } +} + +/** + * The run finished and its summary has aged out of retention. 409 and never a + * re-execution: a key whose run completed is spent, and the fact that nobody + * can still read the OUTCOME does not make the WORK un-done. + */ +export class IdempotentStartAlreadySettledError extends DoStatusError { + readonly status = 409; + readonly reason: { + readonly code: 'IDEMPOTENT_START_ALREADY_SETTLED'; + readonly runId: string; + }; + + constructor(reservation: StartReservation) { + super( + `run '${reservation.runId}' for this idempotency key already completed and its summary has expired`, + ); + this.name = 'IdempotentStartAlreadySettledError'; + this.reason = { + code: 'IDEMPOTENT_START_ALREADY_SETTLED', + runId: reservation.runId, + }; + } +} + +/** A malformed idempotency key or reservation request — the caller's to fix. */ +export class InvalidStartIdempotencyRequestError extends DoStatusError { + readonly status = 400; + readonly reason: { readonly code: 'INVALID_START_IDEMPOTENCY_REQUEST' }; + + constructor(message: string) { + super(message); + this.name = 'InvalidStartIdempotencyRequestError'; + this.reason = { code: 'INVALID_START_IDEMPOTENCY_REQUEST' }; + } +} + +/** + * A start carried a key onto a deployment whose host never wired the + * reservation store. + * + * 503, and never a silent pass-through. Honouring the request without a + * reservation would answer an exactly-once REQUEST with at-least-once + * BEHAVIOUR, which is worse than refusing: the caller would have every reason + * to believe a retry is safe, and no way to find out it is not. The condition + * is a wiring fault an operator fixes, so it reads as transient rather than as + * the caller's mistake. + */ +export class StartIdempotencyUnsupportedError extends DoStatusError { + readonly status = 503; + readonly reason: { readonly code: 'IDEMPOTENT_START_UNSUPPORTED' }; + + constructor() { + super( + 'idempotent starts are not configured on this deployment — wire a StartIdempotencyStore before accepting idempotencyKey', + ); + this.name = 'StartIdempotencyUnsupportedError'; + this.reason = { code: 'IDEMPOTENT_START_UNSUPPORTED' }; + } +} + +/** Failures a start surface may reject while admitting or resolving a key. */ +export type StartReservationRefusal = + | StartReservationOwnerMismatchError + | StartReservationTargetMismatchError + | IdempotentStartPendingError + | IdempotentStartUnresolvableError + | IdempotentStartAlreadySettledError + | StartIdempotencyUnsupportedError + | InvalidStartIdempotencyRequestError; + +/** + * Recognize the five reservation-decision refusals: + * + * - `IDEMPOTENT_START_OWNER_MISMATCH` (403) + * - `IDEMPOTENT_START_TARGET_MISMATCH` (409) + * - `IDEMPOTENT_START_PENDING` (503) + * - `IDEMPOTENT_START_UNRESOLVABLE` (409) + * - `IDEMPOTENT_START_ALREADY_SETTLED` (409) + * + * The guard also recognizes `IDEMPOTENT_START_UNSUPPORTED` keyed admission + * (503) and malformed reservation input (400, + * `INVALID_START_IDEMPOTENCY_REQUEST`). + * + * Unsupported wiring remains in this guard because the keyed request is + * intentionally refused at admission instead of being executed without its + * exactly-once guarantee. `StartReservationUnreadableError` is deliberately + * outside it: unreadable storage is an integrity or availability failure, not + * a reservation decision, and propagates through the shared `DoStatusError` + * renderer with `IDEMPOTENT_START_UNREADABLE` and its 503 status. A false + * answer never permits an unkeyed start: a `StartReservationUnreadableError` + * must still propagate as 503. + */ +export function isStartReservationRefusal( + error: unknown, +): error is StartReservationRefusal { + return ( + error instanceof StartReservationOwnerMismatchError || + error instanceof StartReservationTargetMismatchError || + error instanceof IdempotentStartPendingError || + error instanceof IdempotentStartUnresolvableError || + error instanceof IdempotentStartAlreadySettledError || + error instanceof StartIdempotencyUnsupportedError || + error instanceof InvalidStartIdempotencyRequestError + ); +} + +/** + * Minimal structural D1 surface — the same posture as SnapshotDatabase, + * ApprovalDatabase, and ExecutionFenceDatabase, so every surface that must + * reserve can hand over the binding it already holds with no cast and no second + * seam. Tests back it with node:sqlite; Workers pass `env.DB`. + */ +export interface StartIdempotencyDatabase { + prepare(query: string): StartIdempotencyStatement; +} + +export interface StartIdempotencyStatement { + bind(...values: unknown[]): StartIdempotencyStatement; + run(): Promise; + all(): Promise<{ results: T[] }>; +} + +/** How a surface is wired to the reservation store, or the typed opt-out. */ +export type StartIdempotencyWiring = StartIdempotencyStore | 'none'; + +/** + * Rows affected by a write, read from D1's `{ meta: { changes } }` envelope. + * Restated here for the same reason execution-fence.ts restates it: every + * surface that reserves must be able to import this module without dragging + * the D1 storage adapter (and @mastra/cloudflare-d1 with it) into its bundle. + * + * That bundle rule is why the imports at the top of this file are what they + * are. Four are import-free leaves (principal-identity, cause-chain, + * do-status-error, path-safe-id) and the fifth, execution-fence, imports only + * leaves and the shared provisioning protocol. Nothing on that graph can cycle + * back here, which matters more than usual: the eight DoStatusError subclasses + * in this module are evaluated at module load, so an import edge that came + * back around would meet a class expression still in its temporal dead zone. + */ +function changesOf(result: unknown): number { + const changes = (result as { meta?: { changes?: number } } | undefined)?.meta + ?.changes; + return typeof changes === 'number' ? changes : 0; +} + +/** + * One reservation store per DATABASE, keyed on the BINDING — the same memo, for + * the same reason, as `executionFenceFor`: the router that reserves, the + * topology that claims, the runtime that settles, and the purge that reaps must + * all be looking at one table, and keying on an env object would hand two + * databases the same store the first time a host mutated env across requests. + */ +const startIdempotencyStores = new WeakMap(); + +export function startIdempotencyFor( + db: StartIdempotencyDatabase, +): StartIdempotencyStore { + const existing = startIdempotencyStores.get(db); + if (existing) return existing; + const store = new StartIdempotencyStore(db); + startIdempotencyStores.set(db, store); + return store; +} + +const STATE_CHECK = START_RESERVATION_STATES.map((state) => `'${state}'`).join( + ', ', +); +const TARGET_CHECK = START_TARGET_KINDS.map((kind) => `'${kind}'`).join(', '); +/** + * Built from the principal vocabulary rather than hand-written, so a kind added + * to `EXECUTION_PRINCIPAL_KINDS` cannot leave this constraint behind. The + * failure a stale literal would cause is not a compile error and not a rejected + * write on an existing deployment: `CREATE TABLE IF NOT EXISTS` is a no-op + * against a table that already exists, so the drift would show up only as an + * INSERT refused on whichever database happened to be created after the new + * kind shipped. + */ +const OWNER_KIND_CHECK = EXECUTION_PRINCIPAL_KINDS.map( + (kind) => `'${kind}'`, +).join(', '); + +/** + * The reservation schema. + * + * The CHECK constraints are load-bearing, not decoration: every compare-and-set + * below is stated as `WHERE ... AND state = ''`, which is only a TOTAL + * decision while the column cannot hold a fourth value. A row hand-edited into + * an unknown state would otherwise be a reservation no CAS can advance and no + * purge can reap — a permanently wedged key. + */ +export const START_IDEMPOTENCY_DDL = `CREATE TABLE IF NOT EXISTS ${START_IDEMPOTENCY_TABLE} ( + key TEXT PRIMARY KEY, + owner_kind TEXT NOT NULL CHECK (owner_kind IN (${OWNER_KIND_CHECK})), + owner_id TEXT NOT NULL, + target_kind TEXT NOT NULL CHECK (target_kind IN (${TARGET_CHECK})), + target_id TEXT NOT NULL, + run_id TEXT NOT NULL, + thread_id TEXT, + state TEXT NOT NULL CHECK (state IN (${STATE_CHECK})), + created_at INTEGER NOT NULL, + updated_at INTEGER NOT NULL + )`; + +/** + * `run_id` is how the RUNTIME finds a reservation (terminal reconcile knows the + * run, never the key) and how the purge pairs a reservation with the snapshot + * it outlived. Without the index both degrade to a table scan on every terminal + * run. + */ +export const START_IDEMPOTENCY_RUN_INDEX_DDL = `CREATE INDEX IF NOT EXISTS ${START_IDEMPOTENCY_TABLE}_run + ON ${START_IDEMPOTENCY_TABLE} (run_id)`; + +/** The purge's own access path: terminal rows past the key-validity horizon. */ +export const START_IDEMPOTENCY_STATE_INDEX_DDL = `CREATE INDEX IF NOT EXISTS ${START_IDEMPOTENCY_TABLE}_state + ON ${START_IDEMPOTENCY_TABLE} (state, updated_at)`; + +interface StartReservationRow { + key?: unknown; + owner_kind?: unknown; + owner_id?: unknown; + target_kind?: unknown; + target_id?: unknown; + run_id?: unknown; + thread_id?: unknown; + state?: unknown; + created_at?: unknown; + updated_at?: unknown; +} + +function isStartReservationState( + value: unknown, +): value is StartReservationState { + return ( + typeof value === 'string' && + (START_RESERVATION_STATES as readonly string[]).includes(value) + ); +} + +function isStartTargetKind(value: unknown): value is StartTargetKind { + return ( + typeof value === 'string' && + (START_TARGET_KINDS as readonly string[]).includes(value) + ); +} + +function isEpochMs(value: unknown): value is number { + return typeof value === 'number' && Number.isFinite(value); +} + +/** + * Project a stored row, or refuse it. + * + * A malformed row throws rather than reading as absent, and that direction is + * deliberate: "there is no reservation" is the answer that STARTS A RUN, so it + * must never be reachable from a row this build cannot parse. The CHECK + * constraints make this unreachable on a database this package created; it + * exists for the one that was hand-edited. + * + * The TIMESTAMPS are in that strict set too, rather than coerced to 0 as an + * unparseable number once was. Neither column is decoration: `updated_at` is + * the horizon the purge measures from, so a corrupt one on a terminal row reads + * as epoch 0 and makes the reservation immediately reapable — which deletes a + * spent key early and turns the next retry of it into a fresh start. It is also + * `pendingSince` on a live claim, where 0 tells an operator a run has been + * starting since 1970. Refusing the row keeps both faults visible as the 503 + * they are. + */ +function reservationFromRow(row: StartReservationRow): StartReservation { + const { + key, + owner_kind: ownerKind, + owner_id: ownerId, + target_kind: targetKind, + target_id: targetId, + run_id: runId, + thread_id: threadId, + state, + created_at: createdAt, + updated_at: updatedAt, + } = row; + if ( + typeof key !== 'string' || + !isExecutionPrincipalKind(ownerKind) || + !isExecutionPrincipalId(ownerId) || + !isStartTargetKind(targetKind) || + typeof targetId !== 'string' || + !isPathSafeId(runId) || + !isStartReservationState(state) || + !isEpochMs(createdAt) || + !isEpochMs(updatedAt) + ) { + throw new StartReservationUnreadableError( + typeof key === 'string' ? key : '(unknown)', + ); + } + return { + key, + owner: { kind: ownerKind, id: ownerId }, + targetKind, + targetId, + runId, + ...(isPathSafeId(threadId) ? { threadId } : {}), + state, + createdAt, + updatedAt, + }; +} + +/** + * A reservation exists but cannot be understood, or the table could not be + * read. 503 and never "no reservation": the absent answer is the one that + * starts a run, so an unreadable store degrades CLOSED exactly as an unreadable + * fence does. + */ +export class StartReservationUnreadableError extends DoStatusError { + readonly status = 503; + readonly reason: { readonly code: 'IDEMPOTENT_START_UNREADABLE' }; + + constructor(key: string, options?: ErrorOptions) { + super(`start reservation '${key}' is not readable`, options); + this.name = 'StartReservationUnreadableError'; + this.reason = { code: 'IDEMPOTENT_START_UNREADABLE' }; + } +} + +/** + * SQLite/D1's "no such table", for THIS store's table: the reservation table is + * created lazily by the first `reserve()`, so its absence means no key has ever + * been used here and there is nothing to find. + * + * The rule itself — bounded, cycle-safe, and matched at the ROOT of the cause + * chain only — lives in cause-chain.ts, shared with the fence store. Root-only + * is the load-bearing half: a missing-table link mentioned part-way down a + * chain describes a fault that merely PASSED this table on its way out (a + * failed migration, an adapter reporting the last thing it saw), and concluding + * "no reservations exist" from that would start a run. + * + * Kept local because every consumer is in this store; the drain inventory uses + * `missingTableReadsEmpty` directly with `START_IDEMPOTENCY_TABLE`. + */ +function isMissingReservationTable(error: unknown): boolean { + return missingTableReadsEmpty(error, START_IDEMPOTENCY_TABLE); +} + +function assertKey(key: unknown): string { + if (!isPathSafeId(key)) { + throw new InvalidStartIdempotencyRequestError( + "idempotencyKey must be a URL-path-safe identifier (letters, digits, '.', '_', '~', '-'; 1-200 chars)", + ); + } + return key; +} + +function assertOwner(owner: unknown): StartReservationOwner { + if (owner === null || typeof owner !== 'object') { + throw new InvalidStartIdempotencyRequestError( + 'reservation owner must be an execution principal', + ); + } + const { kind, id } = owner as { kind?: unknown; id?: unknown }; + if (!isExecutionPrincipalKind(kind) || !isExecutionPrincipalId(id)) { + throw new InvalidStartIdempotencyRequestError( + 'reservation owner must be an execution principal', + ); + } + return { kind, id }; +} + +export interface StartIdempotencyStoreOptions { + /** Injectable clock for `created_at`/`updated_at` (tests, fixtures). */ + now?: () => number; + /** Schema readiness override, for a host that owns its own migrations. */ + ready?: () => Promise; +} + +/** + * The deployment's start reservations, over the SAME D1 database its snapshots, + * its resource owners, and its execution fence live in — so a reservation + * cannot be separated from the run it reserves by any binding mistake. + */ +export class StartIdempotencyStore { + readonly #db: StartIdempotencyDatabase; + readonly #now: () => number; + readonly #ready: () => Promise; + + constructor( + db: StartIdempotencyDatabase, + options: StartIdempotencyStoreOptions = {}, + ) { + this.#db = db; + this.#now = options.now ?? Date.now; + if (options.ready) { + this.#ready = options.ready; + } else { + let ready: Promise | undefined; + this.#ready = () => { + ready ??= this.#createSchema().catch((error: unknown) => { + ready = undefined; + throw error; + }); + return ready; + }; + } + } + + /** + * Decide which run this key means, creating the reservation if it is new. + * + * INSERT OR IGNORE then READ BACK — the resource-ownership idiom, and the + * only shape that is correct without a transaction: the insert is the race, + * the read-back is the result, and every caller that lost the insert reads + * the winner's row rather than its own intention. Two callers can therefore + * never both believe they created the reservation, whatever order their + * statements interleave in, and neither needs to know it raced. + * + * This is the ONE method that may create the table. A reservation is the + * first write anything here makes, so the schema belongs on its path and + * nowhere else — in particular not on `read`, where lazy DDL would turn the + * deployment's read-only drain inventory into a write. + */ + async reserve( + request: StartReservationRequest, + ): Promise { + const key = assertKey(request.key); + const owner = assertOwner(request.owner); + if (!isStartTargetKind(request.targetKind)) { + throw new InvalidStartIdempotencyRequestError( + `target kind must be one of ${START_TARGET_KINDS.join(', ')}`, + ); + } + if (!isPathSafeId(request.targetId)) { + throw new InvalidStartIdempotencyRequestError( + 'target id must be a URL-path-safe identifier', + ); + } + // The thread is the agent run's ADDRESS, so requiring it for agents and + // rejecting it for workflows is not tidiness: an agent reservation without + // one is a run a retry can never reach, and a workflow reservation WITH one + // is a second, silently divergent copy of an address that is already + // derivable from (workflowId, runId). + if (request.targetKind === 'agent') { + if (!isPathSafeId(request.threadId)) { + throw new InvalidStartIdempotencyRequestError( + 'an agent start reservation requires a URL-path-safe threadId', + ); + } + } else if (request.threadId !== undefined) { + throw new InvalidStartIdempotencyRequestError( + 'threadId applies only to agent start reservations', + ); + } + await this.#ready(); + const candidateRunId = request.mintRunId(); + if (!isPathSafeId(candidateRunId)) { + throw new InvalidStartIdempotencyRequestError( + 'the host minted a runId that is not URL-path-safe', + ); + } + const now = this.#now(); + const inserted = changesOf( + await this.#db + .prepare( + `INSERT OR IGNORE INTO ${START_IDEMPOTENCY_TABLE} + (key, owner_kind, owner_id, target_kind, target_id, run_id, + thread_id, state, created_at, updated_at) + VALUES (?, ?, ?, ?, ?, ?, ?, 'reserved', ?, ?)`, + ) + .bind( + key, + owner.kind, + owner.id, + request.targetKind, + request.targetId, + candidateRunId, + request.threadId ?? null, + now, + now, + ) + .run(), + ); + const stored = await this.read(key); + if (stored === undefined) { + // The row was written and is already gone: the purge cannot reach a row + // this young, so this is a storage fault, not a reservation state. + throw new StartReservationUnreadableError(key); + } + // Owner FIRST, always. A caller that is not the owner must learn nothing + // about the target, and checking the target first would leak it through the + // choice of refusal. + if (stored.owner.kind !== owner.kind || stored.owner.id !== owner.id) { + throw new StartReservationOwnerMismatchError(key); + } + if ( + stored.targetKind !== request.targetKind || + stored.targetId !== request.targetId + ) { + throw new StartReservationTargetMismatchError(key, stored); + } + // Both signals must agree before this caller believes it created the row. + // `changes` alone would trust the adapter's bookkeeping; the id comparison + // alone would trust that two mints never collide. Requiring both makes the + // false answer the SAFE one — a creator misread as a replayer takes the + // replay path and converges, while the reverse would start a second run. + return { + reservation: stored, + created: inserted > 0 && stored.runId === candidateRunId, + }; + } + + /** + * Take the claim: `reserved` -> `started`, for this exact run. + * + * ONE conditional UPDATE, and the whole cross-isolate serialization of the + * feature. Exactly one caller changes a row; every other caller sees zero + * changes and must go and find out what the winner did rather than starting + * anything. The `run_id` predicate rides along so a claim can never land on a + * row that was rewritten underneath it. + * + * Never creates the table: a claim can only ever follow a reserve, which did. + */ + async claim(key: string, runId: string): Promise { + return this.#casState(key, runId, 'reserved', 'started'); + } + + /** + * Give the claim back: `started` -> `reserved`, for this exact run. + * + * The ONLY backwards transition, and it exists for exactly one caller: a + * start the EXECUTION FENCE refused. That refusal is special because it is + * provably pre-execution — the fence is read before the run lock and before + * any storage write — so the claim it consumed bought nothing and holding on + * to it would manufacture an UNRESOLVABLE reservation out of an operator + * action. Leaving the claim taken would mean a deployment that drained, + * migrated, and reopened had permanently poisoned every key that happened to + * be in flight. + * + * It is deliberately NOT used for other start failures. Anything that reached + * the runtime's execution path may have taken effect, and a rollback there + * would hand the next retry a fresh run — the exact double-charge this whole + * module exists to prevent. + */ + async release(key: string, runId: string): Promise { + return this.#casState(key, runId, 'started', 'reserved'); + } + + /** + * Terminal reconcile, keyed by RUN rather than by key: the runtime observes a + * run reaching a terminal state and has no idea which key (if any) named it. + * + * Idempotent by construction (`state <> 'terminal'`), so the several places a + * run can reach terminal — completing, failing, being cancelled, timing out — + * can all call it without coordinating, and a re-entry after a crash is a + * no-op rather than a conflict. + * + * Returns the number of reservations settled, which is 0 for the overwhelming + * majority of runs (nobody used a key) and 1 for the rest. + */ + async settleRun(runId: string): Promise { + if (!isPathSafeId(runId)) return 0; + try { + return changesOf( + await this.#db + .prepare( + `UPDATE ${START_IDEMPOTENCY_TABLE} + SET state = 'terminal', updated_at = ? + WHERE run_id = ? AND state <> 'terminal'`, + ) + .bind(this.#now(), runId) + .run(), + ); + } catch (error) { + // A database with no reservation table has no reservation to settle. Any + // other fault is real and must not be mistaken for "nothing to do". + if (isMissingReservationTable(error)) return 0; + throw new StartReservationUnreadableError(runId, { cause: error }); + } + } + + /** + * Read one reservation. A PURE read: no lazy DDL, no upsert, nothing that + * would make consulting a key mutate the database — which is what lets a + * read-only drain inventory and every replay path use it freely, and what + * keeps a read-only replica or a revoked-write incident a degrade rather than + * an outage. + * + * An absent TABLE reads as an absent reservation, because on a deployment + * where no key has ever been used those are the same fact. + */ + async read(key: string): Promise { + const safeKey = assertKey(key); + let rows: StartReservationRow[]; + try { + rows = ( + await this.#db + .prepare( + `SELECT key, owner_kind, owner_id, target_kind, target_id, run_id, + thread_id, state, created_at, updated_at + FROM ${START_IDEMPOTENCY_TABLE} WHERE key = ?`, + ) + .bind(safeKey) + .all() + ).results; + } catch (error) { + if (isMissingReservationTable(error)) return undefined; + throw new StartReservationUnreadableError(safeKey, { cause: error }); + } + const row = rows[0]; + return row === undefined ? undefined : reservationFromRow(row); + } + + /** Create the table and its two access paths. Only `reserve()` reaches this. */ + async #createSchema(): Promise { + await this.#db.prepare(START_IDEMPOTENCY_DDL).run(); + await this.#db.prepare(START_IDEMPOTENCY_RUN_INDEX_DDL).run(); + await this.#db.prepare(START_IDEMPOTENCY_STATE_INDEX_DDL).run(); + } + + /** + * Every reservation naming one of these runs, for the purge and the + * inventory. Chunked by the caller; this method binds exactly what it is + * given, so a caller must respect D1's 100-parameter statement limit. + */ + async reservationsForRuns( + runIds: readonly string[], + ): Promise { + const safeRunIds = runIds.filter((runId) => isPathSafeId(runId)); + if (safeRunIds.length === 0) return []; + const placeholders = safeRunIds.map(() => '?').join(', '); + let rows: StartReservationRow[]; + try { + rows = ( + await this.#db + .prepare( + `SELECT key, owner_kind, owner_id, target_kind, target_id, run_id, + thread_id, state, created_at, updated_at + FROM ${START_IDEMPOTENCY_TABLE} WHERE run_id IN (${placeholders})`, + ) + .bind(...safeRunIds) + .all() + ).results; + } catch (error) { + if (isMissingReservationTable(error)) return []; + throw new StartReservationUnreadableError('(run lookup)', { + cause: error, + }); + } + return rows.map((row) => reservationFromRow(row)); + } + + async #casState( + key: string, + runId: string, + from: StartReservationState, + to: StartReservationState, + ): Promise { + const safeKey = assertKey(key); + if (!isPathSafeId(runId)) { + throw new InvalidStartIdempotencyRequestError( + 'reservation runId must be a URL-path-safe identifier', + ); + } + try { + return ( + changesOf( + await this.#db + .prepare( + `UPDATE ${START_IDEMPOTENCY_TABLE} + SET state = ?, updated_at = ? + WHERE key = ? AND run_id = ? AND state = ?`, + ) + .bind(to, this.#now(), safeKey, runId, from) + .run(), + ) > 0 + ); + } catch (error) { + // No table means no reservation, so no transition happened — which is + // exactly what `false` says, and the caller's replay path handles it. + if (isMissingReservationTable(error)) return false; + throw new StartReservationUnreadableError(safeKey, { cause: error }); + } + } +} + +// --------------------------------------------------------------------------- +// The replay resolver — the state machine's OBSERVABLE half. +// +// One function rather than one per surface, because the workflow router and the +// agent topology must answer identically: a caller that retried a workflow +// start and a caller that retried an agent start are asking the same question +// about the same table, and two implementations of "what happened to the first +// one?" is two chances for one of them to answer "nothing did" when something +// had. What differs between surfaces is only WHERE a run's persisted state and +// its liveness are read from, which is exactly what the injected surface says. +// --------------------------------------------------------------------------- + +/** The two surface-specific reads the replay decision needs. */ +export interface IdempotentStartSurface { + /** + * The reserved run's persisted state, or undefined when nothing has been + * persisted yet. This is the FIRST question asked on every replay, because a + * persisted run makes every other branch moot: the work happened, its outcome + * is readable, and the honest answer to the retry is that outcome. + */ + persisted(reservation: StartReservation): Promise; + /** + * Whether the reserved run is executing RIGHT NOW — asked only when nothing + * is persisted, and only to separate "still working" from "died holding the + * claim". Never a timer (see the module header). + * + * KNOWN WINDOW — the claim-to-dispatch gap. The winning CAS lands on the + * Worker, and the object that would report the run live only learns about it + * one dispatch later. A concurrent retry probing inside that gap is told + * UNRESOLVABLE for a run that is about to execute perfectly well. + * + * That is a FALSE ALARM, never a lost or duplicated run: the claim still + * stands, the winner still executes, and the caller's next retry replays the + * persisted summary. Closing it would take moving the claim into the target + * object itself, so that `started` becomes observable only from inside the + * body that is already executing. It is deliberately NOT closed with a grace + * period: a bound short enough to cover a dispatch is indistinguishable from + * the timer this design rejected, and once a timer exists somebody will grow + * it to cover a slow run and re-open the double-charge it was rejected for. + */ + live(reservation: StartReservation): Promise; +} + +/** What a surface must do next, once the reservation has been resolved. */ +export type IdempotentStartDecision = + | { + /** Nobody has started this key's run: proceed, using THIS runId. */ + kind: 'start'; + reservation: StartReservation; + } + | { + /** The run already exists: answer with its persisted state, unchanged. */ + kind: 'replay'; + reservation: StartReservation; + persisted: TPersisted; + }; + +/** + * Resolve an idempotency key into "start this run" or "replay that one". + * + * The order of the checks below IS the semantics: + * + * 1. Reserve. A brand-new key that also wins the claim is the only path that + * starts anything. + * 2. Persisted state, BEFORE the reservation's own state. A run that persisted + * is answerable whatever the reservation says, and reading the row's state + * first would let a stale `started` refuse a retry whose run is sitting + * right there, finished. + * 3. `reserved` with nothing persisted means the first caller died between the + * insert and the claim, having executed nothing — so this caller may take + * the claim and proceed with the SAME runId. That convergence is what makes + * a crashed reservation self-healing instead of a wedged key. + * 4. `started` with nothing persisted is the only ambiguous state in the + * system, and the liveness probe is what resolves it. + * 5. `terminal` with nothing persisted is a completed run whose summary aged + * out. Spent, never re-run. + * + * Throws the applicable structured taxonomy errors; a surface renders them + * through `doErrorResponse` (or its router's equivalent) with no re-mapping. + * Across the public keyed-start surface the eight codes are the five decision + * refusals — IDEMPOTENT_START_OWNER_MISMATCH (403), + * IDEMPOTENT_START_TARGET_MISMATCH (409), IDEMPOTENT_START_PENDING + * (503), IDEMPOTENT_START_UNRESOLVABLE (409), and + * IDEMPOTENT_START_ALREADY_SETTLED (409) — plus IDEMPOTENT_START_UNSUPPORTED + * (503), INVALID_START_IDEMPOTENCY_REQUEST (400), and + * IDEMPOTENT_START_UNREADABLE (503). + */ +export async function beginIdempotentStart( + store: StartIdempotencyStore, + request: StartReservationRequest, + surface: IdempotentStartSurface, + fence?: ExecutionFenceWiring, +): Promise> { + const { reservation, created } = await store.reserve(request); + if (created && (await store.claim(reservation.key, reservation.runId))) { + return { kind: 'start', reservation }; + } + const decision = await resolveExistingReservation( + store, + reservation, + surface, + ); + if (decision.kind === 'replay') { + await rebindProofRun(fence, decision.reservation); + } + return decision; +} + +/** + * Re-assert a proof-only fence's binding to the run this key already made. + * + * The binding is written by `RunnerRuntime.start`, which a REPLAY never + * reaches — so without this, a proof run whose fence lost its `proof_run_id` + * (the fence was moved away and back onto the same key while its run survived) + * would be a run the deployment can read but can no longer RESUME: proof-only + * admits existing work only for `proof_run_id`, and there would be none. + * + * Every guard lives in `recordProofRun`'s own CAS, which is why this can be + * unconditional and best-effort: it changes nothing unless the fence is in + * proof-only under EXACTLY this key with the slot empty or already holding this + * run. A different key, a different state, a different proof run — all are zero + * rows and no harm. A failure is swallowed rather than raised, because the + * caller is being handed the persisted state of a run that already happened, + * and refusing that read would answer a successful retry with an error while + * changing nothing about the run. + */ +async function rebindProofRun( + fence: ExecutionFenceWiring | undefined, + reservation: StartReservation, +): Promise { + if (fence === undefined || fence === 'none') return; + try { + // The reservation KEY is passed as the fence's PROOF KEY: proof-only + // nominates one idempotency key, and `admitsRunStart` admits the start + // carrying it — so the two identifiers are the same string by construction, + // and a rebind for any other key is the zero-row no-op described above. + await fence.recordProofRun(reservation.key, reservation.runId); + } catch (error) { + console.error( + JSON.stringify({ + type: 'start-reservation-proof-rebind-failed', + key: reservation.key, + runId: reservation.runId, + error: error instanceof Error ? error.message : String(error), + }), + ); + } +} + +async function resolveExistingReservation( + store: StartIdempotencyStore, + observed: StartReservation, + surface: IdempotentStartSurface, +): Promise> { + const persisted = await surface.persisted(observed); + if (persisted !== undefined) { + return { kind: 'replay', reservation: observed, persisted }; + } + if (observed.state === 'reserved') { + if (await store.claim(observed.key, observed.runId)) { + return { kind: 'start', reservation: observed }; + } + // The claim was taken between the read and here. Re-read rather than + // assuming: the winner may already have persisted, in which case the right + // answer is its outcome and not a refusal. + const current = await store.read(observed.key); + if (current === undefined || current.runId !== observed.runId) { + // The reservation vanished or was replaced under us. Refusing is the only + // safe answer — a caller that retries gets a clean reserve, while + // silently starting here would start a run under an id nothing reserved. + throw new StartReservationUnreadableError(observed.key); + } + return resolveClaimedReservation(current, surface); + } + return resolveClaimedReservation(observed, surface); +} + +async function resolveClaimedReservation( + reservation: StartReservation, + surface: IdempotentStartSurface, +): Promise> { + if (reservation.state === 'terminal') { + throw new IdempotentStartAlreadySettledError(reservation); + } + if (await surface.live(reservation)) { + throw new IdempotentStartPendingError(reservation); + } + throw new IdempotentStartUnresolvableError(reservation); +} + +/** + * Give back a claim that the EXECUTION FENCE refused, then re-throw. + * + * Homed here, beside the state machine, rather than written out at each start + * site: the rollback is only correct for this one error family, and a copy that + * widened its catch — to "any start failure", say — would hand the next retry a + * fresh run after a start that may well have executed. Keeping the predicate + * and the CAS in one function is what stops that widening from being a one-line + * edit somebody makes in a hurry. + * + * THE CLASS INVARIANT THIS RELIES ON. Giving a claim back is only sound for a + * failure that provably executed NOTHING, and that is a property of the two + * fence refusal codes rather than of the JavaScript class carrying them: + * + * EXECUTION_FENCED is authored at a gate — the run object's start + * route, before any of its own reads or writes, + * and `RunnerRuntime.#assertStartFence`, before + * the run lock and before core mints anything. + * EXECUTION_FENCE_UNREADABLE is authored by the fence READ that fronts those + * same gates, which is even earlier. + * + * Neither code is reachable from anywhere past the point of execution, so a + * refusal carrying one is pre-execution wherever it was observed. + * + * Which is why the predicate is `isExecutionFenceRefusal` — the widened one, + * which admits the wire rebuild — and NOT `instanceof ExecutionFencedError`. + * Both of this function's callers sit on the far side of a Durable Object + * boundary in every DO-backed host: the run object throws, `doErrorResponse` + * renders, and `doSummary` (or the agent topology's `errorFrom`) rebuilds a + * `RunRouteError` carrying the same status and the same structured reason but + * not the same class. An instanceof-only test would answer "not a fence + * refusal" for exactly the deployments this rollback exists to protect, and a + * drained-then-reopened deployment would find every key that was in flight + * permanently stuck at UNRESOLVABLE. In-process hosts (a `{ storage }` runtime + * wired straight into the router) do throw the class, so both shapes are live + * and one predicate has to cover them. + * + * The rollback itself is best-effort: it runs while the deployment is already + * refusing to execute, so its own failure must not replace the fence's refusal + * with a storage error the caller cannot act on. A rollback that does not land + * leaves an UNRESOLVABLE reservation — recoverable by investigation — while a + * swallowed fence refusal would leave the caller believing the deployment is + * broken rather than fenced. + */ +export async function rollbackFencedStart( + store: StartIdempotencyStore, + key: string, + runId: string, + error: unknown, +): Promise { + if (isExecutionFenceRefusal(error)) { + try { + await store.release(key, runId); + } catch (rollbackError) { + console.error( + JSON.stringify({ + type: 'start-reservation-rollback-failed', + key, + runId, + error: + rollbackError instanceof Error + ? rollbackError.message + : String(rollbackError), + }), + ); + } + } + throw error; +} + +/** + * Resolve a surface's reservation wiring, refusing a key the host cannot + * honour. + * + * `undefined` is admitted alongside `'none'` for the same reason + * `readExecutionFence` admits it — a wiring that arrives through an object the + * host may not have populated — and means the same thing: no store. What it + * does NOT mean is "ignore the key", which is why this throws rather than + * returning undefined. A start carrying an idempotency key onto an unwired + * deployment is a request this host cannot answer truthfully. + */ +export function requireStartIdempotency( + wiring: StartIdempotencyWiring | undefined, +): StartIdempotencyStore { + if (wiring === undefined || wiring === 'none') { + throw new StartIdempotencyUnsupportedError(); + } + return wiring; +} diff --git a/packages/flowsafe/src/do-runner/thread-do.test.ts b/packages/flowsafe/src/do-runner/thread-do.test.ts index e03a0a9..33ea61b 100644 --- a/packages/flowsafe/src/do-runner/thread-do.test.ts +++ b/packages/flowsafe/src/do-runner/thread-do.test.ts @@ -24,7 +24,10 @@ class TestThread extends ThreadDurableObject { this.builds += 1; this.events?.push('build'); if (this.buildError) throw this.buildError; - return init({ storage: new InMemoryStore() }); + return init( + { storage: new InMemoryStore() }, + { startIdempotency: 'none', executionFence: 'none' }, + ); } protected route(_request: Request, scope: ThreadScope): Promise { diff --git a/packages/flowsafe/src/execution-entry-matrix.test.ts b/packages/flowsafe/src/execution-entry-matrix.test.ts new file mode 100644 index 0000000..8e1a82b --- /dev/null +++ b/packages/flowsafe/src/execution-entry-matrix.test.ts @@ -0,0 +1,1445 @@ +// SPDX-License-Identifier: Apache-2.0 +// THE EXECUTION-ENTRY MATRIX: every way work can begin or continue on this +// deployment, the fence predicate that polices it, and proof — by driving the +// real surface in all four fence states — that it polices it the way the table +// says. +// +// WHY A MATRIX, AND WHY HERE. The fence is not one check; it is a dozen, spread +// across eight modules that never import each other. Each of those modules +// tests its own gate, and every one of those tests passes on a deployment with +// an entry NOBODY gated: a missing check looks exactly like an absent feature +// until an operator closes the fence and a run starts anyway. What no +// per-module suite can hold is the LIST. This file is that list's +// machine-readable home. +// +// ADDING AN ENTRY. Any new surface that mints a run, resumes one, authors +// standing work, or executes queued work belongs in ENTRIES below, in the same +// change that adds it. Pick its predicate from the four: +// +// admitsRunStart a MINT. Refused from `draining` on, because a +// drain that keeps minting never ends. In +// proof-only, admitted only when the start carries +// the nominated idempotency key. +// admitsExistingRun work on a run that ALREADY exists — resume, +// approval decide, signal delivery. Admitted +// through a drain, because finishing these is what +// the drain is waiting for. In proof-only, only the +// nominated run. +// admitsWorkAuthoring standing configuration that ARMS future work — a +// schedule created or resumed, an objective set, a +// due fire claimed. `open` only; nothing nominates +// it in proof-only. +// admitsDrainableExecution already-owned queued work — a task body, a +// dispatch pass, a webhook ingress. Drains, then +// stops; nothing nominates it either. +// +// Reads — status, inventory, list, observe — and the admin routes are UNGATED +// in every state by design. A surface that only reads does not belong here. +// +// HOW THE PROOF WORKS. The expectation is never written down per entry. For +// each state the table's declared predicate is EVALUATED — the real exported +// function, on the real fence reading — and the surface is then driven and +// required to agree. Declaring the wrong predicate fails, and so does a gate +// that drifts to a different one, because the two sides of the comparison come +// from different places. +// +// Proof-only is driven TWICE where the entry is nominatable: once carrying the +// nomination the fence names, once not. That second probe is what separates the +// predicate PAIRS — across the other three states admitsRunStart is +// indistinguishable from admitsWorkAuthoring, and admitsExistingRun from +// admitsDrainableExecution. + +import { Mastra } from '@mastra/core'; +import type { Agent } from '@mastra/core/agent'; +import type { NotificationsStorage } from '@mastra/core/notifications'; +import { InMemoryStore } from '@mastra/core/storage'; +import { describe, expect, it } from 'vitest'; +import { z } from 'zod'; + +import { openSqlite, sqliteUnitDatabase } from '../test-support/sqlite.js'; +import type { ActorContext, ApprovalActor } from './approval-api/index.js'; +import { + ApprovalService, + InMemoryApprovalStore, + InMemoryResourceOwnershipStore, +} from './approval-api/index.js'; +import { BackgroundTaskHost } from './background-tasks/index.js'; +import type { + DurableObjectRunOwnershipStore, + ExecutionFenceDatabase, + ExecutionFenceReading, + ExecutionFenceState, + RunnerRuntime, + StartIdempotencyDatabase, +} from './do-runner/index.js'; +import { + admitsDrainableExecution, + admitsExistingRun, + admitsRunStart, + admitsWorkAuthoring, + createHostPubSub, + DEPLOYMENT_IDENTITY_HEADER, + DurableObjectRunner, + EXECUTION_PRINCIPAL_HEADER, + ExecutionFenceStore, + init, + StartIdempotencyStore, +} from './do-runner/index.js'; +import { createObjectiveRouter, type ObjectiveStore } from './goals/index.js'; +import { + createThreadTopology, + type ThreadNamespaceLike, +} from './host-kit/index.js'; +import { + createScheduleRouter, + createScheduleTargetPolicy, + createScheduleTick, + D1SchedulesStorage, + type ScheduleDatabase, + scheduleWithCreatorRole, +} from './schedules/index.js'; +import { + createWebhookRouter, + InMemorySubscriptionStoreFactory, + SIGNAL_PROVIDER_HOST_INSTANCE_NAME, + type SignalProviderAdapter, + SignalProviderHost, + type SignalProviderHostState, + type SignalProviderHostWiring, + type SubscriptionStoreFactory, +} from './signal-providers/index.js'; +import { + createNotificationDispatchTick, + createThreadSignalRoutes, +} from './signals/index.js'; + +const STATES: readonly ExecutionFenceState[] = [ + 'open', + 'draining', + 'migration-locked', + 'proof-only', +]; + +const PROOF_KEY = 'proof-key-1'; +const TEST_IDENTITY_SECRET = 'matrix-deployment-identity-secret-0001'; +const THREAD_ID = 'acme_t1'; + +/** Which of the four exported admission predicates an entry is declared under. */ +type PredicateName = + | 'admitsRunStart' + | 'admitsExistingRun' + | 'admitsWorkAuthoring' + | 'admitsDrainableExecution'; + +/** + * The declared predicate, evaluated on a real reading. + * + * `nomination` is what proof-only would have to name for this entry to be + * admitted — an idempotency key for a mint, a runId for work on an existing + * run — and it is `undefined` on the probe that deliberately does not carry it. + */ +function admits( + predicate: PredicateName, + reading: ExecutionFenceReading, + nomination: string | undefined, +): boolean { + switch (predicate) { + case 'admitsRunStart': + return admitsRunStart(reading, nomination); + case 'admitsExistingRun': + return admitsExistingRun(reading, nomination); + case 'admitsWorkAuthoring': + return admitsWorkAuthoring(reading); + case 'admitsDrainableExecution': + return admitsDrainableExecution(reading); + } +} + +type Admission = 'admitted' | 'refused'; + +/** What a prepared entry can do once the fence has moved. */ +interface Prepared { + /** + * What proof-only must name for this entry to be admitted, if anything. An + * entry with no nomination is never admitted in proof-only, and its + * nominated probe asserts exactly that rather than a duplicate. + */ + readonly nomination?: string; + /** Drive the production entry. `carry` supplies the nomination when true. */ + invoke(carry: boolean): Promise; +} + +/** One execution entry. */ +interface Entry { + /** How it reads in a failure message. */ + readonly name: string; + /** The module whose gate this is. */ + readonly module: string; + /** The predicate the gate must behave as. */ + readonly predicate: PredicateName; + /** + * Build the surface with the fence still OPEN, so any prerequisite (a + * suspended run, a filed approval, a due schedule) is created the way + * production creates it. The fence moves only after this returns. + */ + prepare(fence: ExecutionFenceStore): Promise; +} + +/** A fresh fence store over its own in-memory database, seeded open. */ +async function openFence(): Promise { + const fence = new ExecutionFenceStore( + sqliteUnitDatabase(openSqlite()) as ExecutionFenceDatabase, + ); + await fence.seed('open'); + return fence; +} + +/** + * Classify a driven surface. + * + * A refusal reaches a caller two ways — a thrown ExecutionFencedError, or a 503 + * carrying `EXECUTION_FENCED` — and both mean the same thing here. Anything + * else that throws is a broken drive, not a gate, so it is re-thrown and fails + * loudly rather than being counted as a refusal. That distinction is the whole + * reason this helper exists: a drive that quietly errored would otherwise + * "prove" every entry perfectly fenced. + */ +async function classify(run: () => Promise): Promise { + let outcome: unknown; + try { + outcome = await run(); + } catch (error) { + const reason = (error as { reason?: { code?: string } } | undefined) + ?.reason; + if (reason?.code === 'EXECUTION_FENCED') return 'refused'; + throw error; + } + // `null` is a ROUTER SAYING "not my path", never an admission. Counting it + // as one would let a drive that addressed the wrong URL report every state + // perfectly open — the exact shape of false pass this matrix exists to + // prevent — so it fails as a broken drive instead. + if (outcome === null) { + throw new Error('the driven router did not handle the request'); + } + // A surface that returns a value rather than a Response answered normally. + if (!(outcome instanceof Response)) return 'admitted'; + if (outcome.ok) return 'admitted'; + const body = (await outcome.json()) as { reason?: { code?: string } }; + if (outcome.status === 503 && body.reason?.code === 'EXECUTION_FENCED') { + return 'refused'; + } + // Every OTHER non-2xx is a broken drive, not a verdict. Reading a 401 or a + // 500 as "admitted" is how a matrix comes to certify gates it never reached: + // the `open` probe expects admission and would pass on the error, leaving + // only the closed states to fail and no clue why. + throw new Error( + `driven surface answered ${String(outcome.status)}, which is neither an admission nor a fence refusal: ${JSON.stringify(body)}`, + ); +} + +// --------------------------------------------------------------------------- +// Shared fixtures +// --------------------------------------------------------------------------- + +let seq = 0; +function nextRunId(): string { + seq += 1; + return `abc_r${String(seq)}`; +} + +/** A workflow whose only step suspends, so a run can be left mid-flight. */ +function gatedRuntime( + fence: ExecutionFenceStore, + storage = new InMemoryStore(), +): RunnerRuntime { + const { createWorkflow, createStep, runtime } = init( + { storage }, + { + executionFence: fence, + // A real reservation store, not `'none'`: the run object refuses to serve + // a runtime that has none while its env carries a DB binding, so the + // opt-out would fail every DO drive below with a wiring error instead of + // a verdict. + startIdempotency: new StartIdempotencyStore( + sqliteUnitDatabase(openSqlite()) as StartIdempotencyDatabase, + ), + }, + ); + const gate = createStep({ + id: 'gate', + inputSchema: z.object({}), + outputSchema: z.object({}), + suspendSchema: z.object({ reason: z.string() }), + resumeSchema: z.object({ go: z.boolean() }), + execute: async ({ resumeData, suspend }) => { + if (!resumeData) return suspend({ reason: 'wait' }); + return {}; + }, + }); + createWorkflow({ + id: 'gated', + inputSchema: z.object({}), + outputSchema: z.object({}), + }) + .then(gate) + .commit(); + return runtime; +} + +/** A D1 double carrying the deployment sentinel the DO hosts verify against. */ +function deploymentIdentityDatabase(): unknown { + const sqlite = openSqlite(); + sqlite.exec( + `CREATE TABLE flowsafe_deployment ( + id INTEGER PRIMARY KEY CHECK (id = 1), + tenant_tag TEXT NOT NULL, + provisioned_at TEXT NOT NULL + )`, + ); + sqlite + .prepare( + 'INSERT INTO flowsafe_deployment (id, tenant_tag, provisioned_at) VALUES (1, ?, ?)', + ) + .run('acme', new Date(0).toISOString()); + return sqliteUnitDatabase(sqlite); +} + +interface RunnerEnv { + storage: InMemoryStore; + fence: ExecutionFenceStore; + owners: DurableObjectRunOwnershipStore; + DEPLOYMENT_TENANT: string; + DEPLOYMENT_IDENTITY_SECRET: string; + DB: unknown; +} + +/** The production run-object host, over the real in-memory ownership registry. */ +class MatrixRunner extends DurableObjectRunner { + protected runOwnership(env: RunnerEnv): DurableObjectRunOwnershipStore { + return env.owners; + } + + protected runLifecycle(): { abandonApprovals: () => Promise } { + return { abandonApprovals: async () => undefined }; + } + + protected build(env: RunnerEnv): RunnerRuntime { + return gatedRuntime(env.fence, env.storage); + } +} + +function matrixRunner(fence: ExecutionFenceStore): MatrixRunner { + return new MatrixRunner(undefined, { + storage: new InMemoryStore(), + fence, + owners: new InMemoryResourceOwnershipStore(), + DEPLOYMENT_TENANT: 'acme', + DEPLOYMENT_IDENTITY_SECRET: TEST_IDENTITY_SECRET, + DB: deploymentIdentityDatabase(), + }); +} + +function runnerRequest(path: string, body: Record): Request { + return new Request(`http://do${path}`, { + method: 'POST', + headers: { + 'content-type': 'application/json', + [DEPLOYMENT_IDENTITY_HEADER]: TEST_IDENTITY_SECRET, + [EXECUTION_PRINCIPAL_HEADER]: JSON.stringify({ + kind: 'human', + id: 'owner-1', + role: 'operator', + }), + }, + body: JSON.stringify(body), + }); +} + +/** An actor context sufficient for the routers that resolve one. */ +function actorContext(): ActorContext { + const actor: ApprovalActor = { id: 'opal', role: 'operator' }; + return { + actor, + principal: { kind: 'human', id: actor.id, role: actor.role }, + resourceOwner: { kind: 'human', id: actor.id }, + service: () => { + throw new Error('unused by the fence gates this matrix drives'); + }, + newRunId: () => nextRunId(), + newThreadId: () => THREAD_ID, + resourceIdFromKey: (key: string) => key, + claimResource: async () => undefined, + releaseResource: async () => undefined, + resourceOwnerFor: async () => undefined, + canAccessResource: async () => true, + canSelfDecide: () => false, + } as unknown as ActorContext; +} + +/** A thread namespace whose delivery always answers 200. */ +function stubThreadNamespace(): ThreadNamespaceLike { + return { + idFromName: (name: string) => name, + get: () => ({ + fetch: async () => new Response(JSON.stringify({ record: {} })), + }), + } as unknown as ThreadNamespaceLike; +} + +function stubTopology(): ReturnType { + return createThreadTopology(stubThreadNamespace(), TEST_IDENTITY_SECRET); +} + +/** The real D1 schedules domain over node:sqlite, with its schema created. */ +async function schedulesDomain(): Promise { + const store = new D1SchedulesStorage( + sqliteUnitDatabase(openSqlite()) as ScheduleDatabase, + ); + await store.init(); + return store; +} + +const TARGET_POLICY = createScheduleTargetPolicy({ + workflows: [{ id: 'wf' }], + agents: [], +}); + +/** + * The minimum agent the thread signal routes need, with an ACTIVE thread run so + * proof-only has something to nominate. + * + * `Agent` is a @mastra/core class the routes only ever call methods on, so a + * structural stand-in is the honest fixture here — the alternative is booting a + * model, which would test the model. + */ +function matrixAgent(activeRunId: string): Agent { + const delivered = { + signal: { id: 's' }, + accepted: Promise.resolve({ action: 'deliver', runId: activeRunId }), + }; + return { + id: 'agent', + __setPubSub: () => undefined, + getMemory: () => ({ saveMessages: async () => undefined }), + getActiveThreadRunId: () => activeRunId, + sendSignal: () => delivered, + sendMessage: () => delivered, + } as unknown as Agent; +} + +/** The thread-DO scope the signal routes run inside. */ +function threadScope(fence: ExecutionFenceStore): unknown { + return { + threadId: THREAD_ID, + actor: { id: 'operator', role: 'operator' }, + principal: { kind: 'human', id: 'operator', role: 'operator' }, + requestedBy: 'operator', + init: { pubsub: createHostPubSub(), executionFence: fence }, + }; +} + +interface ProviderEnv { + factory: SubscriptionStoreFactory; + fence: ExecutionFenceStore; + DEPLOYMENT_TENANT: string; + DEPLOYMENT_IDENTITY_SECRET: string; + DB: unknown; +} + +class MatrixProviderHost extends SignalProviderHost { + protected build(env: ProviderEnv): SignalProviderHostWiring { + const provider: SignalProviderAdapter = { + id: 'poller', + buildNotification: () => ({ + source: 'poller', + kind: 'poll', + summary: 'poll', + }), + pollForDeliveries: async () => [], + }; + return { + store: env.factory.store(), + topology: stubTopology(), + providers: [provider], + executionFence: env.fence, + }; + } +} + +/** An in-memory thread-state domain for the objective router. */ +function objectiveStore(): ObjectiveStore { + const raw = new Map(); + const key = (threadId: string, type: string) => `${threadId}::${type}`; + return { + getState: async (args: { threadId: string; type: string }) => + raw.get(key(args.threadId, args.type)) as T | undefined, + setState: async (args: { + threadId: string; + type: string; + value: unknown; + }) => { + raw.set(key(args.threadId, args.type), args.value); + }, + deleteState: async (args: { threadId: string; type: string }) => { + raw.delete(key(args.threadId, args.type)); + }, + }; +} + +// --------------------------------------------------------------------------- +// THE ENTRIES +// --------------------------------------------------------------------------- + +const ENTRIES: readonly Entry[] = [ + { + name: 'RunnerRuntime.start', + module: 'do-runner/runtime.ts — the closure guarantee for every mint', + predicate: 'admitsRunStart', + prepare: async (fence) => { + const runtime = gatedRuntime(fence); + return { + nomination: PROOF_KEY, + invoke: (carry) => + classify(() => + runtime.start('gated', { + runId: nextRunId(), + inputData: {}, + ...(carry ? { idempotencyKey: PROOF_KEY } : {}), + }), + ), + }; + }, + }, + { + name: 'RunnerRuntime.resume', + module: 'do-runner/runtime.ts — the closure guarantee for every re-entry', + predicate: 'admitsExistingRun', + prepare: async (fence) => { + const runtime = gatedRuntime(fence); + const runId = nextRunId(); + await runtime.start('gated', { runId, inputData: {} }); + return { + nomination: runId, + invoke: () => + classify(() => + runtime.resume('gated', runId, { + step: 'gate', + resumeData: { go: true }, + requestedBy: 'reviewer-1', + requestedByKind: 'human', + }), + ), + }; + }, + }, + { + name: 'run object POST /runs', + module: + 'do-runner/durable-object.ts — ahead of the recovery journal and the owner reservation', + predicate: 'admitsRunStart', + prepare: async (fence) => { + const runner = matrixRunner(fence); + return { + nomination: PROOF_KEY, + invoke: (carry) => + classify(() => + runner.fetch( + runnerRequest('/runs', { + workflowId: 'gated', + runId: nextRunId(), + inputData: {}, + ...(carry ? { idempotencyKey: PROOF_KEY } : {}), + }), + ), + ), + }; + }, + }, + { + name: 'run object POST /:workflow/:run/resume', + module: 'do-runner/durable-object.ts — ahead of the per-run operation lock', + predicate: 'admitsExistingRun', + prepare: async (fence) => { + const runner = matrixRunner(fence); + const runId = nextRunId(); + await runner.fetch( + runnerRequest('/runs', { workflowId: 'gated', runId, inputData: {} }), + ); + return { + nomination: runId, + invoke: () => + classify(() => + runner.fetch( + runnerRequest(`/runs/gated/${runId}/resume`, { + step: 'gate', + resumeData: { go: true }, + requestedBy: 'reviewer-1', + requestedByKind: 'human', + }), + ), + ), + }; + }, + }, + { + name: 'ApprovalService.decide', + module: 'approval-api/service.ts — commits the decision, then resumes', + predicate: 'admitsExistingRun', + prepare: async (fence) => { + const runId = nextRunId(); + const store = new InMemoryApprovalStore(); + const at = new Date(0).toISOString(); + await store.create({ + id: 'apr-matrix', + workflowId: 'gated', + runId, + title: 'matrix', + connectors: [], + priority: 'normal', + status: 'pending', + createdAt: at, + updatedAt: at, + }); + const service = new ApprovalService({ store, executionFence: fence }); + return { + nomination: runId, + invoke: () => + classify(() => + service.decide( + 'apr-matrix', + { decision: 'approve' }, + { id: 'reviewer-1', role: 'reviewer' }, + ), + ), + }; + }, + }, + { + name: 'thread object POST /signal', + module: 'signals/thread-do-routes.ts — delivery into an existing run', + predicate: 'admitsExistingRun', + prepare: async (fence) => { + const runId = nextRunId(); + const routes = createThreadSignalRoutes({ + resolveAgent: () => matrixAgent(runId), + resolveResourceId: () => 'acme_owner', + }); + return { + nomination: runId, + invoke: () => + classify(() => + routes( + new Request('http://thread/signal', { + method: 'POST', + headers: { 'content-type': 'application/json' }, + body: JSON.stringify({ contents: 'nudge' }), + }), + threadScope(fence) as never, + ), + ), + }; + }, + }, + { + name: 'schedule router create', + module: 'schedules/router.ts — authoring a standing fire', + predicate: 'admitsWorkAuthoring', + prepare: async (fence) => { + const router = createScheduleRouter({ + resolve: async () => actorContext(), + store: await schedulesDomain(), + targetPolicy: TARGET_POLICY, + validateThreadTarget: async () => undefined, + executionFence: fence, + }); + return { + invoke: () => + classify(() => + router( + new Request('http://host/api/schedules', { + method: 'POST', + headers: { 'content-type': 'application/json' }, + body: JSON.stringify({ + workflowId: 'wf', + cron: '*/5 * * * *', + inputData: {}, + }), + }), + ), + ), + }; + }, + }, + { + name: 'schedule tick claim', + module: 'schedules/tick.ts — one fence read per pass, before any CAS claim', + predicate: 'admitsWorkAuthoring', + prepare: async (fence) => { + const store = await schedulesDomain(); + // Through the same helper the router authors with: the tick re-checks + // the creator's role at fire time, and a schedule with none is skipped + // rather than fired — which would read as a fence refusal it is not. + await store.createSchedule( + scheduleWithCreatorRole( + { + id: 'schedule_a', + target: { type: 'workflow', workflowId: 'wf' }, + cron: '* * * * *', + status: 'active', + nextFireAt: 0, + createdAt: 0, + updatedAt: 0, + }, + 'operator', + ), + ); + let fired = 0; + const tick = createScheduleTick({ + store, + targetPolicy: TARGET_POLICY, + start: async ({ runId }) => { + fired += 1; + return { runId }; + }, + status: async () => undefined, + executionFence: fence, + now: () => 1_000, + }); + return { + invoke: async () => { + fired = 0; + await tick(); + // A fenced pass does NOTHING — it never reaches the CAS, because a + // claim it will not run consumes the fire (the claim advances + // nextFireAt) and the fenced runtime then refuses the start. The tick + // runs on an alarm, so it degrades by doing nothing rather than by + // refusing; the work it did is the only honest signal. + return fired > 0 ? 'admitted' : 'refused'; + }, + }; + }, + }, + { + name: 'objective router PUT', + module: 'goals/objective-routes.ts — authoring a standing instruction', + predicate: 'admitsWorkAuthoring', + prepare: async (fence) => { + const router = createObjectiveRouter({ + resolve: async () => actorContext(), + store: objectiveStore(), + validateThreadTarget: async () => undefined, + executionFence: fence, + }); + return { + invoke: () => + classify(() => + router( + new Request(`http://host/api/threads/${THREAD_ID}/goal`, { + method: 'PUT', + headers: { 'content-type': 'application/json' }, + body: JSON.stringify({ objective: 'ship it' }), + }), + ), + ), + }; + }, + }, + { + name: 'webhook ingress', + module: 'signal-providers/webhook-route.ts — after signature verification', + predicate: 'admitsDrainableExecution', + prepare: async (fence) => { + const router = createWebhookRouter({ + providers: { + test: { + id: 'test', + verifyWebhookSignature: () => true, + extractResourceIds: () => [], + buildNotification: () => ({ + source: 'test', + kind: 'k', + summary: 's', + }), + }, + }, + subscriptions: new InMemorySubscriptionStoreFactory().store(), + topology: stubTopology(), + secretForProvider: () => 'webhook-secret', + executionFence: fence, + }); + return { + invoke: () => + classify(() => + router( + new Request('http://host/api/signal-providers/test/webhook', { + method: 'POST', + headers: { 'content-type': 'application/json' }, + body: JSON.stringify({ id: 'evt-1' }), + }), + ), + ), + }; + }, + }, + { + name: 'provider host poll', + module: + 'signal-providers/host-do.ts — one fence read per pass, before any provider runs', + predicate: 'admitsDrainableExecution', + prepare: async (fence) => { + const host = new MatrixProviderHost( + { + id: { name: SIGNAL_PROVIDER_HOST_INSTANCE_NAME }, + } as SignalProviderHostState, + { + factory: new InMemorySubscriptionStoreFactory(), + fence, + DEPLOYMENT_TENANT: 'acme', + DEPLOYMENT_IDENTITY_SECRET: TEST_IDENTITY_SECRET, + DB: deploymentIdentityDatabase(), + }, + ); + return { invoke: () => classify(() => host.poll()) }; + }, + }, + { + name: 'notification dispatch tick', + module: 'signals/notification-dispatch.ts — before the due read', + predicate: 'admitsDrainableExecution', + prepare: async (fence) => { + let listed = 0; + const storage = { + listDueNotifications: async () => { + listed += 1; + return []; + }, + } as unknown as NotificationsStorage; + const tick = createNotificationDispatchTick({ + storage, + topology: stubTopology(), + resolveContext: () => actorContext(), + executionFence: fence, + }); + return { + invoke: async () => { + listed = 0; + await tick(); + // The gate sits BEFORE the due read, so whether the inbox was + // consulted at all is what the pass admitted or refused. Like the + // schedule tick this runs on an alarm and never throws. + return listed > 0 ? 'admitted' : 'refused'; + }, + }; + }, + }, + { + name: 'background task enqueue', + module: + 'background-tasks/host.ts — a drain still accepts, a lock refuses new rows', + predicate: 'admitsDrainableExecution', + prepare: async (fence) => { + const pubsub = createHostPubSub(); + const host = new BackgroundTaskHost({ + mastra: new Mastra({ storage: new InMemoryStore(), pubsub }), + pubsub, + executors: {}, + executionFence: fence, + }); + // Booted while the fence is still open, exactly as a host boots before an + // operator drains it: `boot()` is deliberately NOT fence-gated (a fenced + // refusal would be memoized forever and would take the read routes down + // with it), so the gate this entry drives is the enqueue's own. + await host.boot(); + return { + invoke: () => + classify(async () => { + seq += 1; + await host.enqueue( + { + toolName: 'longResearch', + toolCallId: `call-${String(seq)}`, + args: {}, + agentId: 'agent-1', + runId: 'abc_r1', + }, + { executor: { execute: async () => ({ done: true }) } }, + ); + }), + }; + }, + }, +]; + +// --------------------------------------------------------------------------- +// The matrix +// --------------------------------------------------------------------------- + +/** + * Every place in `src/` that consults an admission predicate, and how the four + * fence states are exercised against it. + * + * THIS IS THE LIST'S ENFORCEMENT. The drives above prove that the gates we know + * about behave correctly; they can say nothing about a gate nobody added and + * nothing about a gate someone deleted. The census below reads the source, so a + * new call site fails until it is written down here — with either the matrix + * entry that drives it, or the suite that already does. + * + * `drivenBy` names a matrix entry above wherever one exists. The three that + * name a test file instead are gates this file cannot reach without a seam that + * production has no other reason to publish: two of them fire inside the + * background-task host's private dispatch path, and one is the wake lane of a + * route whose other arm IS driven here. Each is exercised across all four + * states in the file named. + */ +const GATE_SITES: ReadonlyArray<{ + file: string; + predicate: PredicateName; + drivenBy: string; +}> = [ + { + file: 'approval-api/service.ts', + predicate: 'admitsExistingRun', + drivenBy: 'ApprovalService.decide', + }, + { + file: 'background-tasks/host.ts', + predicate: 'admitsDrainableExecution', + drivenBy: 'background task enqueue', + }, + { + file: 'background-tasks/host.ts', + predicate: 'admitsDrainableExecution', + // #attemptDispatching: whether this instance subscribes and claims at all. + drivenBy: 'background-tasks/host.test.ts', + }, + { + file: 'background-tasks/host.ts', + predicate: 'admitsDrainableExecution', + // The wrapped executor: the task BODY, gated per dispatch so a worker that + // started while open cannot run one after a transition. + drivenBy: 'background-tasks/host.test.ts', + }, + { + file: 'do-runner/durable-object.ts', + predicate: 'admitsRunStart', + drivenBy: 'run object POST /runs', + }, + { + file: 'do-runner/durable-object.ts', + predicate: 'admitsExistingRun', + drivenBy: 'run object POST /:workflow/:run/resume', + }, + { + file: 'do-runner/runtime.ts', + predicate: 'admitsRunStart', + drivenBy: 'RunnerRuntime.start', + }, + { + file: 'do-runner/runtime.ts', + predicate: 'admitsExistingRun', + drivenBy: 'RunnerRuntime.resume', + }, + { + file: 'goals/objective-routes.ts', + predicate: 'admitsWorkAuthoring', + drivenBy: 'objective router PUT', + }, + { + file: 'schedules/router.ts', + predicate: 'admitsWorkAuthoring', + drivenBy: 'schedule router create', + }, + { + file: 'schedules/tick.ts', + predicate: 'admitsWorkAuthoring', + drivenBy: 'schedule tick claim', + }, + { + file: 'signal-providers/host-do.ts', + predicate: 'admitsDrainableExecution', + drivenBy: 'provider host poll', + }, + { + file: 'signal-providers/webhook-route.ts', + predicate: 'admitsDrainableExecution', + drivenBy: 'webhook ingress', + }, + { + file: 'signals/notification-dispatch.ts', + predicate: 'admitsDrainableExecution', + drivenBy: 'notification dispatch tick', + }, + { + file: 'signals/thread-do-routes.ts', + predicate: 'admitsExistingRun', + drivenBy: 'thread object POST /signal', + }, + { + file: 'signals/thread-do-routes.ts', + predicate: 'admitsExistingRun', + // handleWake's own check, for the wake path it owns. + drivenBy: 'signals/thread-do-routes.test.ts', + }, +]; + +/** + * The files whose `admits*` mentions are not call sites: the module that + * DEFINES the predicates, and the barrel that re-exports them. + */ +const NOT_GATE_FILES = ['do-runner/execution-fence.ts', 'do-runner/index.ts']; + +type SourceFileSystem = { + existsSync(path: string | URL): boolean; + readdirSync( + path: string, + options: { withFileTypes: true }, + ): Array<{ name: string; isDirectory: () => boolean }>; + readFileSync(path: string, encoding: string): string; +}; + +function sourceFileSystem(): SourceFileSystem { + return ( + globalThis as { + process?: { getBuiltinModule?: (id: string) => unknown }; + } + ).process?.getBuiltinModule?.('node:fs') as SourceFileSystem; +} + +type SourceFile = { readonly file: string; readonly source: string }; + +function sourceRoot(): string { + // This module's OWN directory, never `process.cwd()`: filtered and root test + // runs use different working directories. + const here = (import.meta as ImportMeta & { url: string }).url; + return new URL('.', here).pathname.replace(/\/$/, ''); +} + +function walkSourceFiles( + root: string, + visit: (sourceFile: SourceFile) => void, +): void { + const fs = sourceFileSystem(); + const walk = (directory: string, prefix: string): void => { + for (const entry of fs.readdirSync(directory, { withFileTypes: true })) { + const absolute = `${directory}/${entry.name}`; + const relative = prefix === '' ? entry.name : `${prefix}/${entry.name}`; + if (entry.isDirectory()) { + walk(absolute, relative); + continue; + } + if ( + !/\.(?:ts|tsx)$/.test(entry.name) || + /\.(?:test|stories)\.(?:ts|tsx)$/.test(entry.name) + ) { + continue; + } + visit({ file: relative, source: fs.readFileSync(absolute, 'utf8') }); + } + }; + walk(root, ''); +} + +/** + * Every `admits*(` call in the package's source, as (file, predicate) pairs. + * + * `process.getBuiltinModule` rather than an import: this package's test + * tsconfig is workers-typed and carries no `@types/node`, so a static `node:` + * specifier does not type-check. This is the schema guard's idiom for reaching + * `node:sqlite`. + */ +function gateCallSites(): Array<{ file: string; predicate: PredicateName }> { + const found: Array<{ file: string; predicate: PredicateName }> = []; + const pattern = + /\badmits(RunStart|ExistingRun|WorkAuthoring|DrainableExecution)\s*\(/g; + walkSourceFiles(sourceRoot(), ({ file, source }) => { + if (NOT_GATE_FILES.includes(file)) return; + for (const match of source.matchAll(pattern)) { + found.push({ + file, + predicate: `admits${match[1] as string}` as PredicateName, + }); + } + }); + return found; +} + +type FenceErrorName = 'ExecutionFencedError' | 'ExecutionFenceUnreadableError'; + +/** + * Every production site that AUTHORS a fence refusal or unreadable-store + * failure. Each row states why the error is constructed before execution can + * have an effect; the source scan below makes a new author fail until its + * boundary is reviewed and recorded here. The census is deliberately lexical: + * a constructor spelling in a comment or string fails loud and asks for review. + * Aliased class names and namespace imports are forbidden so lexical coverage + * cannot be bypassed without first changing this test. + */ +const FENCE_ERROR_AUTHORS: ReadonlyArray<{ + file: string; + error: FenceErrorName; + anchor: string; + beforeExecutionEffect: string; +}> = [ + { + file: 'approval-api/service.ts', + error: 'ExecutionFencedError', + anchor: 'async #assertDecidable', + beforeExecutionEffect: + 'The admission check runs before decide() mutates the approval or resumes its run.', + }, + { + file: 'background-tasks/host.ts', + error: 'ExecutionFencedError', + anchor: '#gated(executor', + beforeExecutionEffect: + 'The executor backstop refuses before calling a tool body when core supplies no suspension seam.', + }, + { + file: 'background-tasks/host.ts', + error: 'ExecutionFencedError', + anchor: 'async enqueue(', + beforeExecutionEffect: + 'The enqueue admission check runs before the manager creates a queued task row.', + }, + { + file: 'do-runner/durable-object.ts', + error: 'ExecutionFencedError', + anchor: 'const startFence =', + beforeExecutionEffect: + 'The start route refuses before source lookup, recovery journalling, owner reservation, or runtime start.', + }, + { + file: 'do-runner/durable-object.ts', + error: 'ExecutionFencedError', + anchor: 'const resumeFence =', + beforeExecutionEffect: + 'The resume route refuses before handing the existing run to runtime.resume().', + }, + { + file: 'do-runner/execution-fence.ts', + error: 'ExecutionFencedError', + anchor: 'export function executionFencedResponse', + beforeExecutionEffect: + 'The response helper only serializes an already-decided refusal and performs no execution effect.', + }, + { + file: 'do-runner/execution-fence.ts', + error: 'ExecutionFenceUnreadableError', + anchor: 'function readingFromRow', + beforeExecutionEffect: + 'Fence-row validation fails closed before any caller can admit execution.', + }, + { + file: 'do-runner/execution-fence.ts', + error: 'ExecutionFenceUnreadableError', + anchor: 'async read(): Promise', + beforeExecutionEffect: + 'A failed fence read becomes unreadable before an admission predicate can run.', + }, + { + file: 'do-runner/execution-fence.ts', + error: 'ExecutionFenceUnreadableError', + anchor: 'async recordProofRun(', + beforeExecutionEffect: + 'A failed proof-binding metadata write becomes unreadable before the runtime starts the run.', + }, + { + file: 'do-runner/runtime.ts', + error: 'ExecutionFencedError', + anchor: 'if (!admitsRunStart(reading, idempotencyKey))', + beforeExecutionEffect: + 'The start admission check refuses before proof binding and engine run creation.', + }, + { + file: 'do-runner/runtime.ts', + error: 'ExecutionFencedError', + anchor: 'if (!(await fence.recordProofRun', + beforeExecutionEffect: + 'A lost proof-binding compare-and-set refuses while engine run creation has not begun.', + }, + { + file: 'do-runner/runtime.ts', + error: 'ExecutionFencedError', + anchor: 'async #assertResumeFence', + beforeExecutionEffect: + 'The resume admission check refuses before the engine continues the existing run.', + }, + { + file: 'signal-providers/host-do.ts', + error: 'ExecutionFencedError', + anchor: 'async poll(): Promise', + beforeExecutionEffect: + 'The poll admission check refuses before any provider is polled or notification is delivered.', + }, +]; + +function fenceErrorAuthorSites(): Array<{ + file: string; + error: FenceErrorName; + line: number; +}> { + const found: Array<{ file: string; error: FenceErrorName; line: number }> = + []; + const pattern = + /\bnew\s+(ExecutionFencedError|ExecutionFenceUnreadableError)\s*\(/g; + walkSourceFiles(sourceRoot(), ({ file, source }) => { + for (const match of source.matchAll(pattern)) { + found.push({ + file, + error: match[1] as FenceErrorName, + line: source.slice(0, match.index).split('\n').length, + }); + } + }); + return found; +} + +const FENCE_ERROR_CENSUS_PATTERNS = [ + { + pattern: + /\b(?:ExecutionFencedError|ExecutionFenceUnreadableError)\s+as\s+\w+/g, + message: + 'import the fence error classes under their own names so the census can see the site, or reword a comment that spells one of these class names followed by `as`', + }, + { + pattern: + /\bnew\s+\w+\.(?:ExecutionFencedError|ExecutionFenceUnreadableError)\s*\(/g, + message: + 'construct the fence error classes under their own imported names so the census can see the site', + }, + { + pattern: + /\bextends\s+(?:\w+\.)*(?:ExecutionFencedError|ExecutionFenceUnreadableError)\b/g, + message: + 'subclassing the fence error classes is not census-visible; author fence refusals with the two classes directly', + }, +] as const; + +function matchesFenceErrorCensusViolation(source: string): boolean { + return FENCE_ERROR_CENSUS_PATTERNS.some(({ pattern }) => + new RegExp(pattern.source, pattern.flags).test(source), + ); +} + +function fenceErrorCensusViolations(): string[] { + const found: string[] = []; + walkSourceFiles(sourceRoot(), ({ file, source }) => { + for (const { pattern, message } of FENCE_ERROR_CENSUS_PATTERNS) { + for (const match of source.matchAll(pattern)) { + const line = source.slice(0, match.index).split('\n').length; + found.push(`${file}:${line}: ${message}: ${match[0]}`); + } + } + }); + return found; +} + +const ANCHOR_WINDOW_LINES = 30; +const sourceLinesByFile = new Map(); + +function anchorDistanceBeforeAuthor( + author: (typeof FENCE_ERROR_AUTHORS)[number], + site: ReturnType[number], +): number | undefined { + let lines = sourceLinesByFile.get(site.file); + if (!lines) { + lines = sourceFileSystem() + .readFileSync(`${sourceRoot()}/${site.file}`, 'utf8') + .split('\n'); + sourceLinesByFile.set(site.file, lines); + } + for ( + let index = site.line - 2; + index >= Math.max(0, site.line - (ANCHOR_WINDOW_LINES + 1)); + index -= 1 + ) { + if (lines[index]?.includes(author.anchor)) { + return site.line - (index + 1); + } + } + return undefined; +} + +function siteKey(site: { file: string; predicate: string }): string { + return `${site.file} :: ${site.predicate}`; +} + +describe('execution-entry matrix', () => { + it('rejects aliases, qualified construction, and subclassing census escapes', () => { + const escapes = [ + "import { ExecutionFencedError as HiddenFenceError } from './do-runner/index.js';", + 'new fence.ExecutionFenceUnreadableError()', + 'class HiddenFenceError extends ExecutionFencedError {}', + 'class HiddenFenceError extends fence.ExecutionFencedError {}', + ]; + + expect(escapes.map(matchesFenceErrorCensusViolation)).toEqual([ + true, + true, + true, + true, + ]); + }); + + it('accounts for every production fence-error author, with a recorded pre-execution justification', () => { + expect( + fenceErrorCensusViolations(), + 'fence-error construction must stay visible to the lexical census', + ).toEqual([]); + const sites = fenceErrorAuthorSites(); + expect(sites).toHaveLength(FENCE_ERROR_AUTHORS.length); + const anchored = new Map< + (typeof sites)[number], + (typeof FENCE_ERROR_AUTHORS)[number] + >(); + for (const site of sites) { + const candidates = FENCE_ERROR_AUTHORS.flatMap((author) => { + if (author.file !== site.file || author.error !== site.error) return []; + const distance = anchorDistanceBeforeAuthor(author, site); + return distance === undefined ? [] : [{ author, distance }]; + }); + const nearest = Math.min(...candidates.map(({ distance }) => distance)); + const authors = candidates.filter(({ distance }) => distance === nearest); + expect( + authors.map(({ author }) => author.anchor), + `${site.file}:${site.line} must have one declared anchor in the preceding ${ANCHOR_WINDOW_LINES} lines`, + ).toHaveLength(1); + const author = authors[0]?.author; + if (author) anchored.set(site, author); + } + for (const author of FENCE_ERROR_AUTHORS) { + const sitesForAnchor = sites.filter( + (site) => anchored.get(site) === author, + ); + expect( + sitesForAnchor.map((site) => site.line), + `${author.file} :: ${author.error} :: ${author.anchor} must anchor one author site`, + ).toHaveLength(1); + expect( + author.beforeExecutionEffect.length, + `${author.file} :: ${author.error} :: ${author.anchor} needs a substantive pre-execution justification`, + ).toBeGreaterThan(40); + } + }); + + it('accounts for every admission call site in the source, and for no site that is gone', () => { + // #given — the drives below can only prove the gates somebody listed. This + // is what catches the gate nobody listed: a new execution entry, or a check + // quietly deleted from an existing one. + const actual = gateCallSites().map(siteKey).sort(); + const declared = GATE_SITES.map(siteKey).sort(); + + // #then — multiset equality both ways. An unlisted call site fails (a new + // entry nobody censused), and so does a listed one that no longer exists (a + // gate removed while its row stayed behind, claiming protection that is no + // longer there). + expect(actual).toEqual(declared); + }); + + it('drives every gate site it claims to, and names a real suite for the rest', () => { + // #given — `drivenBy` is either a matrix entry above or a test file. A + // typo in the first would silently turn a driven gate into a delegated one. + const entryNames = new Set(ENTRIES.map((entry) => entry.name)); + const fs = sourceFileSystem(); + const here = (import.meta as ImportMeta & { url: string }).url; + + // #then + for (const site of GATE_SITES) { + if (site.drivenBy.endsWith('.test.ts')) { + expect( + fs.existsSync(new URL(site.drivenBy, here)), + `${siteKey(site)} delegates to missing suite '${site.drivenBy}'`, + ).toBe(true); + continue; + } + expect( + entryNames, + `${siteKey(site)} claims to be driven by '${site.drivenBy}', which is not a matrix entry`, + ).toContain(site.drivenBy); + } + + // #and — every matrix entry drives at least one real gate site, so an + // entry whose surface stopped consulting the fence cannot keep passing on + // an admission it now grants unconditionally. + const driven = new Set(GATE_SITES.map((site) => site.drivenBy)); + for (const entry of ENTRIES) { + expect( + driven, + `matrix entry '${entry.name}' drives no censused gate site`, + ).toContain(entry.name); + } + }); + + it('names every entry exactly once and exercises all four predicates', () => { + // #given — the LIST is the product here: a duplicate name would let two + // rows describe one gate while a third gate went unlisted, and a predicate + // with no entry left would make its family vacuously satisfied. + const names = ENTRIES.map((entry) => entry.name); + + // #then + expect(new Set(names).size).toBe(names.length); + expect(new Set(ENTRIES.map((entry) => entry.predicate)).size).toBe(4); + }); + + for (const entry of ENTRIES) { + for (const state of STATES) { + it(`${entry.name} behaves as ${entry.predicate} under '${state}'`, async () => { + // #given — the surface built while the fence is still open, so its + // prerequisites are created the way production creates them. + const fence = await openFence(); + const prepared = await entry.prepare(fence); + + // #when — the fence moves to the state under test. + if (state !== 'open') { + await fence.transition({ + expected: 'open', + next: state, + ...(state === 'proof-only' ? { proofKey: PROOF_KEY } : {}), + }); + } + const reading = await fence.read(); + + // #then — carrying no nomination, the declared predicate decides, and + // the driven surface must agree with it. The expectation comes from the + // real exported predicate, never from a hand-written table, so an entry + // declared under the wrong one fails here. + expect(await prepared.invoke(false)).toBe( + admits(entry.predicate, reading, undefined) ? 'admitted' : 'refused', + ); + }); + } + + it(`${entry.name} answers its proof-only nomination as ${entry.predicate} (${entry.module})`, async () => { + // #given — the probe that separates the predicate PAIRS. Across open, + // draining, and migration-locked, admitsRunStart is indistinguishable + // from admitsWorkAuthoring and admitsExistingRun from + // admitsDrainableExecution; only the nominated proof-only case tells them + // apart. + const fence = await openFence(); + const prepared = await entry.prepare(fence); + await fence.transition({ + expected: 'open', + next: 'proof-only', + proofKey: PROOF_KEY, + }); + if ( + prepared.nomination !== undefined && + entry.predicate === 'admitsExistingRun' + ) { + // For work on an EXISTING run the nomination is the run itself, bound + // the way an admitted proof-only start binds it. + await fence.recordProofRun(PROOF_KEY, prepared.nomination); + } + const reading = await fence.read(); + const expected = admits(entry.predicate, reading, prepared.nomination) + ? 'admitted' + : 'refused'; + + // #then — an entry with a nomination is admitted when it carries it; one + // without is refused however it is driven, which is the whole meaning of + // "nothing nominates authoring or queued execution". + expect(await prepared.invoke(true)).toBe(expected); + if (prepared.nomination === undefined) expect(expected).toBe('refused'); + }); + } +}); diff --git a/packages/flowsafe/src/goals/objective-routes.test.ts b/packages/flowsafe/src/goals/objective-routes.test.ts index 04a5af0..c877fdd 100644 --- a/packages/flowsafe/src/goals/objective-routes.test.ts +++ b/packages/flowsafe/src/goals/objective-routes.test.ts @@ -1,12 +1,15 @@ // SPDX-License-Identifier: Apache-2.0 -// The goal objective HTTP surface (createObjectiveRouter): the P6-lite gate ORDER -// (401 -> ownership -> role -> size/body/field/cap -> audit -> persist), each -// fail-closed, plus the set/get/update/clear round-trip (byte-identical to core's -// Agent goal methods), the maxRuns host cap, and the GOAL_REQUEST_CONTEXT_KEY -// no-collision reservation — over mock resolve + store seams. +// The goal objective HTTP surface (createObjectiveRouter): bounded ingestion +// gate order (401 -> ownership -> role -> size/body/field/cap -> audit -> +// persist). Each fails closed. The suite also covers the set/get/update/clear +// round-trip, byte-identical to core's Agent goal methods, the maxRuns host +// cap, and the GOAL_REQUEST_CONTEXT_KEY no-collision reservation over mock +// resolver and store seams. import { describe, expect, it, vi } from 'vitest'; +import { openSqlite, sqliteUnitDatabase } from '../../test-support/sqlite.js'; + import type { ActorContext, ApprovalActor } from '../approval-api/index.js'; import { BREAKWATER_ACTOR_KEY, @@ -15,6 +18,11 @@ import { BREAKWATER_ISOLATION_SCOPE_KEY, BREAKWATER_WORKFLOW_SCOPE_KEY, } from '../do-runner/breakwater-keys.js'; +import { + type ExecutionFenceDatabase, + type ExecutionFenceState, + ExecutionFenceStore, +} from '../do-runner/index.js'; import { RunRouteError } from '../host-kit/index.js'; import { createObjectiveRouter as createObjectiveRouterImpl, @@ -26,12 +34,24 @@ import { const OWNED_THREAD = 'acme_t1'; +/** + * The router under test with the two options every case would otherwise repeat + * defaulted. `executionFence: 'none'` is the honest wiring for the in-memory + * ObjectiveStore these cases use — there is no database to fence — and the + * fence cases below pass a real store, so nothing here weakens their gate. + */ function createObjectiveRouter( - options: Omit & - Partial>, + options: Omit< + ObjectiveRouterOptions, + 'validateThreadTarget' | 'executionFence' + > & + Partial< + Pick + >, ) { return createObjectiveRouterImpl({ ...options, + executionFence: options.executionFence ?? 'none', validateThreadTarget: options.validateThreadTarget ?? (async () => undefined), }); @@ -108,7 +128,7 @@ interface GoalRecord { prompt?: string; } -describe('createObjectiveRouter — the P6-lite ingestion gate', () => { +describe('createObjectiveRouter — bounded ingestion gate', () => { it.each([ { maxRunsCap: 0 }, { maxRunsCap: 1.5 }, @@ -379,7 +399,7 @@ describe('createObjectiveRouter — the P6-lite ingestion gate', () => { }); }); -describe('createObjectiveRouter — maxRuns host cap (DL-007)', () => { +describe('createObjectiveRouter — maxRuns host cap', () => { it('rejects a maxRuns above the host cap and audits it (never clamps)', async () => { const { store, raw } = memoryStore(); const events: ObjectiveAuditEvent[] = []; @@ -594,7 +614,7 @@ describe('createObjectiveRouter — set / get / update / clear round-trip', () = }); }); -describe('GOAL_REQUEST_CONTEXT_KEY reservation (DL-018 no-collision pin)', () => { +describe('GOAL_REQUEST_CONTEXT_KEY reservation', () => { // The keys #requestContextFor mints on a leg: the workflow-scope + isolation- // scope base (runtime.ts) plus the grant/actor keys the provider merges over // them. ALL are the breakwater 'breakwater.*' namespace; the goal key is @@ -613,11 +633,11 @@ describe('GOAL_REQUEST_CONTEXT_KEY reservation (DL-018 no-collision pin)', () => }); it('pins the mirrored value to the core dist declaration (drift guard)', () => { - // GOAL_REQUEST_CONTEXT_KEY is not exports-reachable (R-001), so the mirror + // GOAL_REQUEST_CONTEXT_KEY is not exports-reachable, so the mirror // cannot be compared via an import — read the pinned core's own .d.ts // declaration instead. A core bump that changes the value (or moves the - // file) fails HERE loudly, per the P9 re-anchor protocol. Builtins load via - // process.getBuiltinModule (the test-support/sqlite.ts idiom) and + // file) fails HERE loudly under the re-anchoring protocol. Builtins load + // via process.getBuiltinModule (the test-support/sqlite.ts idiom) and // import.meta.url is cast, so this workers-typed program never sees a // node: import it cannot type. const getBuiltin = ( @@ -693,3 +713,137 @@ describe('createObjectiveRouter internal errors', () => { } }); }); + +describe('createObjectiveRouter and the deployment execution fence', () => { + async function fenceAt( + state: ExecutionFenceState, + ): Promise { + const fence = new ExecutionFenceStore( + sqliteUnitDatabase(openSqlite()) as ExecutionFenceDatabase, + ); + await fence.seed(state); + return fence; + } + + function unreadableFence(): ExecutionFenceStore { + // Storage that faults on every query — NOT the "no such table" a pre-0.20 + // database answers with, which legitimately reads as open. + return new ExecutionFenceStore({ + prepare: () => ({ + bind: () => ({ + bind: () => { + throw new Error('unreachable'); + }, + run: () => Promise.reject(new Error('D1_ERROR: network')), + all: () => Promise.reject(new Error('D1_ERROR: network')), + }), + run: () => Promise.reject(new Error('D1_ERROR: network')), + all: () => Promise.reject(new Error('D1_ERROR: network')), + }), + } as unknown as ExecutionFenceDatabase); + } + + it('degrades a mutation closed with 503 when the fence cannot be read', async () => { + // #given + const { store, raw } = memoryStore(); + const router = createObjectiveRouter({ + resolve: async () => actorContext('operator'), + store, + executionFence: unreadableFence(), + }); + + // #then + const res = await router( + req('PUT', OWNED_THREAD, { objective: 'ship it' }), + ); + expect(res?.status).toBe(503); + expect(await res?.json()).toMatchObject({ + reason: { code: 'EXECUTION_FENCE_UNREADABLE' }, + }); + expect(raw.size).toBe(0); + }); + + it('refuses every objective-ARMING mutation once draining, and writes nothing', async () => { + // #given — a standing objective is authored work: it is what the agent loop + // re-reads to decide it should run again. + const { store, raw } = memoryStore(); + const events: ObjectiveAuditEvent[] = []; + const router = createObjectiveRouter({ + resolve: async () => actorContext('operator'), + store, + audit: (event) => { + events.push(event); + }, + executionFence: await fenceAt('draining'), + }); + + // #when / #then + for (const request of [ + req('PUT', OWNED_THREAD, { objective: 'ship it' }), + req('PATCH', OWNED_THREAD, { status: 'paused' }), + ]) { + const res = await router(request); + expect(res?.status).toBe(503); + expect(await res?.json()).toMatchObject({ + reason: { code: 'EXECUTION_FENCED', state: 'draining' }, + }); + } + expect(raw.size).toBe(0); + expect( + events.filter((event) => event.reason === 'execution-fenced'), + ).toHaveLength(2); + }); + + it('keeps CLEAR open in every state, because clearing removes future work', async () => { + // #given — an objective already authored, on a deployment locked down for a + // migration. `clear` is the exemption schedule pause/delete get: it takes + // standing work AWAY, which is the direction a drain is going, and refusing + // it would leave an operator unable to quiet an agent they are migrating. + const { store, raw } = memoryStore(); + const events: ObjectiveAuditEvent[] = []; + const open = createObjectiveRouter({ + resolve: async () => actorContext('operator'), + store, + executionFence: await fenceAt('open'), + }); + expect( + (await open(req('PUT', OWNED_THREAD, { objective: 'ship it' })))?.status, + ).toBe(200); + expect(raw.size).toBe(1); + + // #when — the same store behind a migration-locked fence. + const locked = createObjectiveRouter({ + resolve: async () => actorContext('operator'), + store, + audit: (event) => { + events.push(event); + }, + executionFence: await fenceAt('migration-locked'), + }); + const res = await locked(req('DELETE', OWNED_THREAD)); + + // #then — accepted, committed, and audited as a committed mutation rather + // than a fenced refusal. + expect(res?.status).toBe(200); + expect(await res?.json()).toEqual({ ok: true }); + expect(raw.size).toBe(0); + expect( + events.filter((event) => event.reason === 'execution-fenced'), + ).toHaveLength(0); + }); + + it('keeps the objective READ open while locked', async () => { + // #given — a read moves nothing, and an operator proving a drain needs it. + const { store } = memoryStore(); + const router = createObjectiveRouter({ + resolve: async () => actorContext('operator'), + store, + executionFence: await fenceAt('migration-locked'), + }); + + // #then + const res = await router(req('GET', OWNED_THREAD)); + expect(res?.status).toBe(200); + expect(await res?.json()).toEqual({ objective: null }); + }); +}); diff --git a/packages/flowsafe/src/goals/objective-routes.ts b/packages/flowsafe/src/goals/objective-routes.ts index febec7e..b6b4385 100644 --- a/packages/flowsafe/src/goals/objective-routes.ts +++ b/packages/flowsafe/src/goals/objective-routes.ts @@ -1,12 +1,12 @@ // SPDX-License-Identifier: Apache-2.0 -// Track F (M-005), CI-M-005-001 — the goal objective HTTP surface (DL-018). +// The goal objective HTTP surface. // // A goal is a durable, thread-scoped OBJECTIVE record (@mastra/core's // GoalObjectiveRecord, stored in the mastra_thread_state domain under type // 'goal') that a Mastra agent's in-loop judge reads to decide whether to keep // working toward it. In the DURABLE path the goal step rebuilds RequestContext // from initData and reads the objective from D1 via resolveGoalStore -> -// readObjective, NOT any in-process registry (DL-018). Verified against the +// readObjective, NOT any in-process registry. Verified against the // on-disk @mastra/core 1.50.0 dist: resolveGoalStore(mastra) is // `mastra.getStorage().getStore('threadState')` and readObjective is // `getState({ threadId, type: 'goal' })`. So this surface writes the D1 domain @@ -15,49 +15,49 @@ // through core's OWN writeObjective/readObjective/clearObjective over the SAME // (threadId, 'goal') key, so the stored shape can never drift from the reader's. // -// An objective is a STANDING INSTRUCTION injected into every future model turn, -// so the write path is an ingestion trust boundary (P6-lite, DL-006). It uses -// the same resource-first authorization rule as the signal router: +// An objective is a STANDING INSTRUCTION injected into every future model +// turn, so the write path is an ingestion trust boundary. It uses the same +// resource-first authorization rule as the signal router: // -// 1. resolve (authenticate and validate actor) -> 401 / 403 -// 2. registry-backed thread ownership -> 404 -// 3. coarse role (RUN_START_ROLES) on MUTATIONS -> 403 (reads stay coarse) -// 4. size cap on the raw body, THEN JSON parse -> 413 / 400 -// 5. body names NO client memory id (assertNoClientMemoryIds) -> 400 -// 6. field allowlist (objective/maxRuns/judge/prompt; status on update) -> 400 -// 7. maxRuns host cap (DL-007) -> 400 +// 1. resolve and validate the actor -> 401 / 403 +// 2. verify registry-backed thread ownership -> 404 +// 3. require RUN_START_ROLES for mutations -> 403 +// 4. cap the raw body, then parse JSON -> 413 / 400 +// 5. reject client-supplied memory ids -> 400 +// 6. allow only objective fields -> 400 +// 7. enforce the host maxRuns cap -> 400 // 8. audit (goal.objective) + persist // // Every MUTATION (set/update/clear) is audited on ACCEPT and on EVERY post-auth -// denial (role 403, malformed target 404, size/body/field/cap 400) — the Track C lesson. -// A GET is audited only on a post-auth denial; a +// denial (role 403, malformed target 404, size/body/field/cap 400), following +// the signal-ingestion lesson. A GET is audited only on a post-auth denial; a // benign successful read is not a standing-instruction write and is not logged. // Pre-auth failures (401 / a resolver throw -> 403) are NOT audited: an // unauthenticated flood must never be able to write the log. // -// maxRuns (DL-007): a requested maxRuns above the host cap is REJECTED (400), +// maxRuns: a requested maxRuns above the host cap is REJECTED (400), // not silently clamped. A caller that asked for 200 evaluations and got 50 would // see mysterious early-stopping — exactly the "your value was quietly replaced" // footgun the run router 400s a client runId to avoid — so the boundary fails // loud instead. The stored record therefore never carries maxRuns above the cap -// (default the core DEFAULT_GOAL_MAX_RUNS, 50). Track F itself starts NO runs: +// (default the core DEFAULT_GOAL_MAX_RUNS, 50). This router starts NO runs: // the deployment run-start budget stays enforced at the existing run-start seam; // bounding maxRuns only bounds how many of those already-budgeted runs one goal // can drive. // -// P8 (goals never mint capability): no route reads or writes requestContext and +// Goals never mint capability: no route reads or writes requestContext and // none names a connector/grant/step. writeObjective is called WITHOUT a // requestContext, so the within-turn GOAL_REQUEST_CONTEXT_KEY surface is never // touched here — the objective payload is content only. // // Write serialization: update is a read-modify-write and clear an unconditional // delete, both at the Worker with NO per-thread serialization — deliberately -// weaker than signals (whose sends ride the thread-DO lease), because DL-018 -// writes D1 directly. Core's own -// Agent.updateObjectiveOptions has the identical non-atomic read-then-write, so -// concurrent mutations are last-write-wins within the deployment; the maxRuns cap is -// enforced on every write, so no interleaving can persist an over-cap value or -// cross an authorization/capability line. +// weaker than signals (whose sends ride the thread-DO lease), because this +// router writes D1 directly. Core's own Agent.updateObjectiveOptions has the +// identical non-atomic read-then-write, so concurrent mutations are +// last-write-wins within the deployment. The maxRuns cap is enforced on every +// write, so no interleaving can persist an over-cap value or cross an +// authorization/capability line. import type { GoalObjectiveRecord } from '@mastra/core/storage'; import { @@ -74,6 +74,13 @@ import { type ApprovalRole, RUN_START_ROLES, } from '../approval-api/index.js'; +import { + admitsWorkAuthoring, + type ExecutionFenceWiring, + executionFencedResponse, + isExecutionFenceRefusal, + readExecutionFence, +} from '../do-runner/index.js'; import { assertNoClientMemoryIds, type BoundThreadTargetValidator, @@ -180,6 +187,19 @@ export interface ObjectiveRouterOptions { * zero denies every non-empty body. Default 16384. */ maxContentBytes?: number; + /** + * The deployment execution fence, or `'none'` for a router with no database + * behind it. A standing objective is authored work — it is what the agent + * loop re-reads to decide it should run again — so SET and UPDATE are + * refused past `open`. CLEAR stays available in every state because it + * removes a standing instruction rather than arming one (the same reason + * schedule pause and delete are exempt), and GET stays available in every + * state. + * + * REQUIRED: this router receives a store facade, not a database, so the host + * is the only place the wiring can happen. See ExecutionFenceWiring. + */ + executionFence: ExecutionFenceWiring; /** Route prefix. Default '/api/threads' (goals mount at `/:threadId/goal`). */ basePath?: string; } @@ -383,7 +403,7 @@ function buildUpdateRecord( export function createObjectiveRouter( options: ObjectiveRouterOptions, ): ObjectiveRouter { - const { resolve, store } = options; + const { executionFence, resolve, store } = options; const roles = options.roles ?? RUN_START_ROLES; const maxRunsCap = positiveSafeInteger( options.maxRunsCap ?? DEFAULT_GOAL_MAX_RUNS, @@ -478,6 +498,22 @@ export function createObjectiveRouter( return json({ error: 'forbidden' }, 403); } + // 3b. The execution fence, after the role gate so a refusal is auditable + // and tells an unauthorized caller nothing. Reads pass in every state, + // and so does `clear`: it REMOVES a standing instruction rather than + // arming one, which is the direction a drain is going — the same reason + // schedule pause/delete are exempt (FENCE_GATED_SCHEDULE_OPERATIONS). + if (isMutation && operation !== 'clear') { + const reading = await readExecutionFence(executionFence); + if (!admitsWorkAuthoring(reading)) { + await audit('rejected', 'execution-fenced'); + return executionFencedResponse( + reading.state, + `objective ${operation}`, + ); + } + } + // 4. GET / CLEAR carry no validated body. if (operation === 'get') { const record = await readObjective(store, threadId); @@ -558,6 +594,17 @@ export function createObjectiveRouter( await auditCommittedMutation(); return json({ objective: built.value }); } catch (error) { + // A fence that could not be READ is not evidence the deployment is open, + // so it degrades closed with its own retryable 503 rather than the + // generic 500 below — an operator must be able to tell a deployment + // that is being migrated from one that is broken. + if (isExecutionFenceRefusal(error)) { + await audit('rejected', 'execution-fence-unreadable'); + return json( + { error: error.message, reason: error.reason }, + error.status, + ); + } if (error instanceof RunRouteError) { // A post-auth denial: the target 404 or a smuggled memory-id 400. await audit( diff --git a/packages/flowsafe/src/host-kit/approval-bridge.test.ts b/packages/flowsafe/src/host-kit/approval-bridge.test.ts index 040d56a..117d347 100644 --- a/packages/flowsafe/src/host-kit/approval-bridge.test.ts +++ b/packages/flowsafe/src/host-kit/approval-bridge.test.ts @@ -2,8 +2,8 @@ // Unit coverage for the host-kit approval bridge: the payload-shape edge cases // of requestedConnectors, the (suspendedAt, resumeCount) capture in // queueApprovalForSuspension, the multi-gate re-queue + fail-closed guard in -// resumeRunWithRequeue (plus its D4 audit signal on a re-queue failure), and -// the D4 self-healing reconcile (reconcileApprovalsForSummary). These are the +// resumeRunWithRequeue (plus its audit signal on a re-queue failure), and the +// self-healing reconcileApprovalsForSummary. These are the // pieces the showcase Worker and dev backend both depend on, so they get // direct tests independent of any workflow. @@ -54,6 +54,9 @@ describe('abandonApprovalsForRun', () => { const events: ApprovalAuditEvent[] = []; const service = new ApprovalService({ store, + // In-memory store, no database to fence against: the opt-out is written down + // rather than defaulted — see ExecutionFenceWiring. + executionFence: 'none', resumeRun, audit: (event) => events.push(event), }); @@ -139,6 +142,7 @@ describe('abandonApprovalsForRun', () => { }; const service = new ApprovalService({ store, + executionFence: 'none', resumeRun: async () => { throw new Error('a fenced decision must never resume'); }, @@ -218,7 +222,7 @@ describe('abandonApprovalsForRun', () => { sqliteUnitDatabase(sqlite) as never, { workflowSnapshotTable: 'mastra_workflow_snapshot' }, ).store(); - const service = new ApprovalService({ store }); + const service = new ApprovalService({ store, executionFence: 'none' }); const [approval] = await queueApprovalForSuspension( service, 'product-launch', @@ -248,7 +252,7 @@ describe('abandonApprovalsForRun', () => { sqliteUnitDatabase(sqlite) as never, { workflowSnapshotTable: 'mastra_workflow_snapshot' }, ).store(); - const service = new ApprovalService({ store }); + const service = new ApprovalService({ store, executionFence: 'none' }); const { record } = await service.createAsPrincipal( { workflowId: 'plain-queue', @@ -294,7 +298,7 @@ describe('abandonApprovalsForRun', () => { await blocked; return originalCreate(record); }; - const service = new ApprovalService({ store }); + const service = new ApprovalService({ store, executionFence: 'none' }); const filing = queueApprovalForSuspension( service, 'product-launch', @@ -345,7 +349,7 @@ describe('requestedConnectors', () => { ['connectors as a non-array', { connectors: 'a' }, []], ['connectors with a non-string element', { connectors: ['a', 2] }, []], ['a valid connectors array', { connectors: ['a', 'b'] }, ['a', 'b']], - // Track A (R-003): an AGENT gate declares no connectors array — the tool the + // An AGENT gate declares no connectors array — the tool the // model called is derived from its (flat or nested) suspend shape. [ 'a FLAT agent gate', @@ -421,7 +425,7 @@ function suspendedSummary( describe('queueApprovalForSuspension', () => { it('persists tool-call scope when an agent payload also carries connectors', async () => { const store = new InMemoryApprovalStore(); - const service = new ApprovalService({ store }); + const service = new ApprovalService({ store, executionFence: 'none' }); const summary: RunSummary = { runId: 'acme_agent-ambiguous', status: 'suspended', @@ -456,7 +460,7 @@ describe('queueApprovalForSuspension', () => { it('captures the suspended step, its (suspendedAt, resumeCount) pair, and connectors', async () => { // #given — a run suspended at gate2 on its second suspension const store = new InMemoryApprovalStore(); - const service = new ApprovalService({ store }); + const service = new ApprovalService({ store, executionFence: 'none' }); const summary = suspendedSummary( 'acme_run-1', 'gate2', @@ -501,7 +505,7 @@ describe('queueApprovalForSuspension', () => { it('files one record PER suspended path when parallel branches suspend together', async () => { // #given — a .parallel() run suspended at TWO gates in one summary const store = new InMemoryApprovalStore(); - const service = new ApprovalService({ store }); + const service = new ApprovalService({ store, executionFence: 'none' }); const summary: RunSummary = { runId: 'acme_run-parallel', status: 'suspended', @@ -553,7 +557,7 @@ describe('queueApprovalForSuspension', () => { if (record.stepPath?.[0] === 'gateA') throw new Error('d1 hiccup'); return originalCreate(record); }; - const service = new ApprovalService({ store }); + const service = new ApprovalService({ store, executionFence: 'none' }); const summary: RunSummary = { runId: 'acme_run-partial', status: 'suspended', @@ -600,6 +604,7 @@ describe('resumeRunWithRequeue', () => { // on a later decision); the same const-with-deferred-ref shape worker.ts uses. const service: ApprovalService = new ApprovalService({ store, + executionFence: 'none', resumeRun: resumeRunWithRequeue(base, () => service, SYSTEM), }); @@ -653,6 +658,7 @@ describe('resumeRunWithRequeue', () => { }); const service: ApprovalService = new ApprovalService({ store, + executionFence: 'none', resumeRun: resumeRunWithRequeue(base, () => service, SYSTEM), }); const { record } = await service.createAsPrincipal( @@ -682,6 +688,7 @@ describe('resumeRunWithRequeue', () => { suspendedSummary('acme_run-agent', 'gate2', ['connector'], 2020, 1, null); const service: ApprovalService = new ApprovalService({ store, + executionFence: 'none', resumeRun: resumeRunWithRequeue(base, () => service, SYSTEM), }); const { record } = await service.createAsPrincipal( @@ -730,7 +737,7 @@ describe('resumeRunWithRequeue', () => { it('fails closed: refuses to re-queue a suspension with no decider', async () => { // #given — the wrapper invoked directly with a record lacking decidedBy const store = new InMemoryApprovalStore(); - const service = new ApprovalService({ store }); + const service = new ApprovalService({ store, executionFence: 'none' }); const base: ResumeRunFn = async () => suspendedSummary('acme_run-3', 'gate2', ['c'], 9, undefined, null); const wrapped = resumeRunWithRequeue(base, () => service, SYSTEM); @@ -755,6 +762,7 @@ describe('resumeRunWithRequeue', () => { suspendedSummary('acme_run-9', 'gate2', ['deploy-conn'], 3030, 1, null); const service: ApprovalService = new ApprovalService({ store, + executionFence: 'none', audit, resumeRun: resumeRunWithRequeue(base, () => service, '', audit), }); @@ -796,7 +804,7 @@ describe('resumeRunWithRequeue', () => { expect(requeue[0]?.detail).not.toHaveProperty('principalKind'); }); - it('emits an audit event and reports resume.ok=false when the post-resume re-queue throws (D4)', async () => { + it('emits an audit event and reports resume.ok=false when the post-resume re-queue throws', async () => { // #given — deciding gate1 durably resumes to a re-suspended gate2, but // the store rejects gate2's filing (a transient D1 failure) — the base // resume has already landed by the time this fires @@ -812,6 +820,7 @@ describe('resumeRunWithRequeue', () => { suspendedSummary('acme_run-4', 'gate2', ['deploy-conn'], 3030, 1, null); const service: ApprovalService = new ApprovalService({ store, + executionFence: 'none', audit, resumeRun: resumeRunWithRequeue(base, () => service, SYSTEM, audit), }); @@ -870,7 +879,7 @@ describe('resumeRunWithRequeue', () => { describe('reconcileApprovalsForSummary', () => { it('refuses to file a suspension with missing requester provenance', async () => { const store = new InMemoryApprovalStore(); - const service = new ApprovalService({ store }); + const service = new ApprovalService({ store, executionFence: 'none' }); const summary = suspendedSummary( 'acme_run-missing-requester', 'gate1', @@ -887,10 +896,10 @@ describe('reconcileApprovalsForSummary', () => { }); it('files a fresh approval for a suspended step with no matching record at all', async () => { - // #given — a run reported suspended, nothing ever queued for it (the D4 - // wedge: the original filing never landed) + // #given — a run reported suspended, nothing ever queued for it (the + // approval reconciliation wedge: the original filing never landed) const store = new InMemoryApprovalStore(); - const service = new ApprovalService({ store }); + const service = new ApprovalService({ store, executionFence: 'none' }); const summary = suspendedSummary( 'acme_run-5', 'gate1', @@ -920,7 +929,7 @@ describe('reconcileApprovalsForSummary', () => { it('preserves the durable execution requester on every reconciled record', async () => { // #given — two gates suspended together, neither ever queued const store = new InMemoryApprovalStore(); - const service = new ApprovalService({ store }); + const service = new ApprovalService({ store, executionFence: 'none' }); const summary: RunSummary = { runId: 'acme_run-parallel-2', status: 'suspended', @@ -956,7 +965,7 @@ describe('reconcileApprovalsForSummary', () => { it('uses an explicitly recovered agent principal for reconcile attribution', async () => { const store = new InMemoryApprovalStore(); - const service = new ApprovalService({ store }); + const service = new ApprovalService({ store, executionFence: 'none' }); const summary = suspendedSummary( 'acme_agent-reconcile', 'gate', @@ -996,7 +1005,7 @@ describe('reconcileApprovalsForSummary', () => { it('does not file when a PENDING record already matches the current fingerprint', async () => { // #given — the normal path already queued gate1's approval const store = new InMemoryApprovalStore(); - const service = new ApprovalService({ store }); + const service = new ApprovalService({ store, executionFence: 'none' }); const summary = suspendedSummary( 'acme_run-6', 'gate1', @@ -1029,7 +1038,7 @@ describe('reconcileApprovalsForSummary', () => { // suspended at the exact fingerprint the decision targeted — the window // between decide() landing and its resume completing const store = new InMemoryApprovalStore(); - const service = new ApprovalService({ store }); + const service = new ApprovalService({ store, executionFence: 'none' }); const summary = suspendedSummary( 'acme_run-7', 'gate1', @@ -1064,7 +1073,7 @@ describe('reconcileApprovalsForSummary', () => { it('files when the only existing record carries a PREVIOUS fingerprint (post-resume re-suspension)', async () => { // #given — gate1's first suspension was queued and decided... const store = new InMemoryApprovalStore(); - const service = new ApprovalService({ store }); + const service = new ApprovalService({ store, executionFence: 'none' }); const firstSuspension = suspendedSummary( 'acme_run-8', 'gate1', @@ -1114,7 +1123,7 @@ describe('reconcileApprovalsForSummary', () => { // (produced e.g. by the raw grant-free resume route re-suspending the // step while the old request still sits open) const store = new InMemoryApprovalStore(); - const service = new ApprovalService({ store }); + const service = new ApprovalService({ store, executionFence: 'none' }); const staleSuspension = suspendedSummary( 'acme_run-11', 'gate1', @@ -1175,7 +1184,11 @@ describe('reconcileApprovalsForSummary', () => { const store = new InMemoryApprovalStore(); const events: ApprovalAuditEvent[] = []; const audit = (event: ApprovalAuditEvent) => events.push(event); - const service = new ApprovalService({ store, audit }); + const service = new ApprovalService({ + store, + audit, + executionFence: 'none', + }); const staleSuspension = suspendedSummary( 'acme_run-10', 'gate1', @@ -1237,7 +1250,7 @@ describe('reconcileApprovalsForSummary', () => { // would produce: by the time supersedeStale's CAS runs, the record has // already left the OPEN set, so that CAS naturally loses. const store = new InMemoryApprovalStore(); - const service = new ApprovalService({ store }); + const service = new ApprovalService({ store, executionFence: 'none' }); const staleSuspension = suspendedSummary( 'acme_run-9', 'gate1', @@ -1297,7 +1310,7 @@ describe('reconcileApprovalsForSummary', () => { // gateA's supersede CAS will lose to a concurrent real decision, gateB's // will succeed. The back-off must be scoped per step, not per call. const store = new InMemoryApprovalStore(); - const service = new ApprovalService({ store }); + const service = new ApprovalService({ store, executionFence: 'none' }); const [staleA] = await queueApprovalForSuspension( service, 'product-launch', @@ -1370,7 +1383,7 @@ describe('reconcileApprovalsForSummary', () => { it('excludes a superseded record from grant derivation even queried at its ORIGINAL fingerprint', async () => { // #given — an approval superseded via reconcile... const store = new InMemoryApprovalStore(); - const service = new ApprovalService({ store }); + const service = new ApprovalService({ store, executionFence: 'none' }); const staleSuspension = suspendedSummary( 'acme_run-12', 'gate1', diff --git a/packages/flowsafe/src/host-kit/do-run-topology.test.ts b/packages/flowsafe/src/host-kit/do-run-topology.test.ts index 729b35a..5388fe9 100644 --- a/packages/flowsafe/src/host-kit/do-run-topology.test.ts +++ b/packages/flowsafe/src/host-kit/do-run-topology.test.ts @@ -50,6 +50,7 @@ describe('createDoRunTopology', () => { start: async () => summary, status: async () => summary, dispatchStatus: async () => summary, + startLiveness: async () => false, resume: async () => summary, resumeRecord: async () => summary, }; diff --git a/packages/flowsafe/src/host-kit/do-run-topology.ts b/packages/flowsafe/src/host-kit/do-run-topology.ts index dc22c9a..8f727bf 100644 --- a/packages/flowsafe/src/host-kit/do-run-topology.ts +++ b/packages/flowsafe/src/host-kit/do-run-topology.ts @@ -53,6 +53,21 @@ export interface DoRunTopology { workflowId: string, runId: string, ): Promise; + /** + * Is a start for this run executing in its Durable Object right now? + * + * createRunRouter's liveness probe, and the reason an idempotent replay can + * tell "still working" from "died holding the claim" without a timer. The run + * object is the only place that can answer: it is addressed by + * `idFromName(workflowId:runId)`, so there is exactly one instance that could + * be running this run, and if that instance says no then nothing is. + * + * An unreachable object reads as NOT live. That is the fail-closed direction + * here — it produces the refusal that asks a human to investigate, whereas a + * default of "live" would answer a permanently broken run with a permanently + * retryable 503. + */ + startLiveness(workflowId: string, runId: string): Promise; /** createRunRouter's `resume` thunk. */ resume( workflowId: string, @@ -95,6 +110,11 @@ export type DoRunStartInput = RunStartInput & { initialState?: unknown; }; +/** The run object's answer to the liveness probe. */ +interface StartLivenessBody { + live?: unknown; +} + export function createDoRunTopology( namespace: RunnerNamespaceLike, deploymentIdentitySecret: string, @@ -138,6 +158,7 @@ export function createDoRunTopology( scheduleId, dispatchId, deadlineMs, + idempotencyKey, }) => { if ((scheduleId === undefined) !== (dispatchId === undefined)) { throw new Error( @@ -154,6 +175,11 @@ export function createDoRunTopology( body: JSON.stringify({ workflowId, runId, + // The key travels on THIS channel and no other: the request + // carries the deployment-identity header, which a tenant request + // bearing one is refused for, so the DO can treat what arrives + // here as the router's own reserved key. + ...(idempotencyKey === undefined ? {} : { idempotencyKey }), ...(scheduleId === undefined ? { inputData, initialState, deadlineMs } : { scheduleId, dispatchId, deadlineMs }), @@ -161,6 +187,23 @@ export function createDoRunTopology( }), ); }, + startLiveness: async (workflowId, runId) => { + // Only an explicit `true` from the run's own object counts as live. + // Everything else — a non-200, an unparseable body, a field of the wrong + // shape — is "this did not tell me the run is running", which is not the + // same as "it is", and treating it as such would turn a broken probe into + // an indefinitely retryable PENDING for a run nobody is executing. + const response = await stub(workflowId, runId).fetch( + `http://do/runs/${workflowId}/${runId}/start-liveness`, + { headers: deploymentIdentityHeaders(deploymentIdentitySecret) }, + ); + if (response.status !== 200) return false; + try { + return ((await response.json()) as StartLivenessBody).live === true; + } catch { + return false; + } + }, // The DO answers 404 for a run it has never seen; the router turns the // undefined into its own 404 rather than leaking the DO's body. status: async (workflowId, runId) => { diff --git a/packages/flowsafe/src/host-kit/flowsafe-worker.test.ts b/packages/flowsafe/src/host-kit/flowsafe-worker.test.ts index ed8dfbd..0383038 100644 --- a/packages/flowsafe/src/host-kit/flowsafe-worker.test.ts +++ b/packages/flowsafe/src/host-kit/flowsafe-worker.test.ts @@ -1160,7 +1160,7 @@ describe('createFlowsafeWorker maintenance duties', () => { }); }); - // Track B: the background-task TTL cleanup as the purge alarm's opt-in duty. + // Background-task TTL cleanup as the purge alarm's opt-in duty. async function seedOldCompletedTask(env: FlowsafeWorkerEnv): Promise { await env.DB.prepare( `CREATE TABLE mastra_background_tasks ( @@ -1185,7 +1185,7 @@ describe('createFlowsafeWorker maintenance duties', () => { // #when await worker.runMaintenanceDuty('purge', env); - // #then — the duty never ran, byte-identical to before Track B + // #then — the duty never ran when the feature was absent const lines = maintenanceLines(logs.lines()); expect(lines[0]).not.toHaveProperty('backgroundTasksCompletedPurged'); expect(logs.errors()).toEqual([]); @@ -1660,7 +1660,7 @@ describe('createFlowsafeWorker storage table prefix', () => { }); }); -describe('createFlowsafeWorker schedule tick duty (Track D)', () => { +describe('createFlowsafeWorker schedule tick duty', () => { it('the tick duty runs only the schedule tick', async () => { // #given a worker with a tick interval + a scheduleTick builder const logs = capturedLogs(); @@ -1776,3 +1776,501 @@ describe('createFlowsafeWorker schedule tick duty (Track D)', () => { ); }); }); + +describe('createFlowsafeWorker approval decisions under the execution fence', () => { + // The store's create() takes a complete record. Requested by someone OTHER + // than the decider, so separation of duties is never what refuses a decide + // in these tests. `stepPath` makes it suspension-bound, i.e. a record whose + // decision genuinely drives a resume. + function pendingRecord(runId: string): ApprovalRecord { + const now = new Date('2026-08-01T00:00:00.000Z').toISOString(); + return { + id: `approval-${runId}`, + workflowId: 'demo-approval', + runId, + stepPath: ['approval'], + title: 'publish launch post', + connectors: [], + priority: 'normal', + status: 'pending', + requestedBy: 'opal', + requestedByKind: 'human', + createdAt: now, + updatedAt: now, + }; + } + + it('refuses a decide with 503 and commits NOTHING while migration-locked', async () => { + // #given — a worker composed exactly as a published host composes it, and a + // pending approval requested by someone OTHER than the decider (so + // separation of duties is not what refuses this). + const worker = makeWorker(); + const { env, ctx } = makeEnv(); + const store = approvalStoreFactoryFor(env.DB).store(); + const { record } = await store.create(pendingRecord('acme_run-fenced')); + + // #and — the control plane locks the deployment through its own route, + // rather than by reaching for a store this test built. What that pins is + // AGREEMENT over the one D1 binding: the route's write and the read + // decide() makes land on the same database, so a worker that fenced its + // approval path against some other fence would fail here. It does not pin + // store IDENTITY — two stores over one binding would agree too — and the + // WeakMap that makes them one instance is pinned separately below. + env.MAINTENANCE_ADMIN_SECRET = 'maintenance-admin-secret-0000000001'; + const locked = await worker.fetch( + new Request('http://host/admin/execution-fence', { + method: 'POST', + headers: { + authorization: `Bearer ${env.MAINTENANCE_ADMIN_SECRET}`, + 'content-type': 'application/json', + }, + body: JSON.stringify({ expected: 'open', next: 'migration-locked' }), + }), + env, + ctx, + ); + expect(locked.status).toBe(200); + + // #when + const response = await worker.fetch( + authed(`http://host/api/approvals/${record.id}/decide`, { + method: 'POST', + body: JSON.stringify({ decision: 'approve' }), + }), + env, + ctx, + ); + + // #then — the taxonomy's retryable refusal, not a 500 and not a silent + // success. This is the case that made #assertDecidable dead code: the + // composer built the service without a fence, so every published host + // decided straight through a migration lock. + expect(response.status).toBe(503); + expect(await response.json()).toMatchObject({ + reason: { code: 'EXECUTION_FENCED', state: 'migration-locked' }, + }); + + // #then — and NOTHING committed. decide() writes the decision and only + // then resumes, so a gate any later than the pre-commit check would leave + // this record durably 'approved' with a resume that 503s behind it, and + // the deployment taking over would inherit a decision with nothing behind + // it. + const stored = await store.get(record.id); + expect(stored).toMatchObject({ status: 'pending' }); + expect(stored).not.toHaveProperty('decidedBy'); + expect(stored).not.toHaveProperty('decision'); + expect(stored).not.toHaveProperty('decidedAt'); + // The whole row is byte-identical to the one that was seeded: not even + // updatedAt moved, so no audit trail and no reviewer notification fired + // either — decide() never reached its transition. + expect(stored).toEqual(record); + }); + + it('lets the approval lifecycle observe the NEXT read after each admin move', async () => { + // #given — the store-sharing invariant, in the only form that is observable + // from outside: the admin route and the approval service must consult one + // fence, per request, in both directions. A service that cached its reading + // at composition time, or that was handed a different store, passes the + // lock-then-refuse case above and fails here — it would keep refusing after + // the operator reopened, stranding every decision on the deployment the + // migration just finished with. + const worker = makeWorker({ + buildResumeRun: () => async () => successSummary('acme_run-reopened'), + }); + const { env, ctx } = makeEnv(); + const store = approvalStoreFactoryFor(env.DB).store(); + const { record } = await store.create(pendingRecord('acme_run-reopened')); + env.MAINTENANCE_ADMIN_SECRET = 'maintenance-admin-secret-0000000001'; + const move = (body: unknown) => + worker.fetch( + new Request('http://host/admin/execution-fence', { + method: 'POST', + headers: { + authorization: `Bearer ${env.MAINTENANCE_ADMIN_SECRET}`, + 'content-type': 'application/json', + }, + body: JSON.stringify(body), + }), + env, + ctx, + ); + const decide = () => + worker.fetch( + authed(`http://host/api/approvals/${record.id}/decide`, { + method: 'POST', + body: JSON.stringify({ decision: 'approve' }), + }), + env, + ctx, + ); + + // #when / #then — locked, and the very next decide sees it. + expect( + (await move({ expected: 'open', next: 'migration-locked' })).status, + ).toBe(200); + expect((await decide()).status).toBe(503); + + // #when / #then — reopened, and the very next decide sees THAT. + expect( + (await move({ expected: 'migration-locked', next: 'open' })).status, + ).toBe(200); + expect((await decide()).status).toBe(200); + await expect(store.get(record.id)).resolves.toMatchObject({ + status: 'approved', + }); + }); + + it('still decides while draining', async () => { + // #given — a drain finishes outstanding work, and a suspended run is + // waiting for exactly this decision. + const worker = makeWorker({ + buildResumeRun: () => async () => successSummary('acme_run-draining'), + }); + const { env, ctx } = makeEnv(); + const store = approvalStoreFactoryFor(env.DB).store(); + const { record } = await store.create(pendingRecord('acme_run-draining')); + env.MAINTENANCE_ADMIN_SECRET = 'maintenance-admin-secret-0000000001'; + await worker.fetch( + new Request('http://host/admin/execution-fence', { + method: 'POST', + headers: { + authorization: `Bearer ${env.MAINTENANCE_ADMIN_SECRET}`, + 'content-type': 'application/json', + }, + body: JSON.stringify({ expected: 'open', next: 'draining' }), + }), + env, + ctx, + ); + + // #when / #then + const response = await worker.fetch( + authed(`http://host/api/approvals/${record.id}/decide`, { + method: 'POST', + body: JSON.stringify({ decision: 'approve' }), + }), + env, + ctx, + ); + expect(response.status).toBe(200); + await expect(store.get(record.id)).resolves.toMatchObject({ + status: 'approved', + }); + }); +}); + +describe('createFlowsafeWorker execution-fence administration', () => { + const ADMIN_SECRET = 'maintenance-admin-secret-0000000001'; + + function fenceRequest(init: { + method: 'GET' | 'POST'; + body?: unknown; + token?: string | null; + }): Request { + const token = init.token === undefined ? ADMIN_SECRET : init.token; + return new Request('http://host/admin/execution-fence', { + method: init.method, + headers: { + ...(token === null ? {} : { authorization: `Bearer ${token}` }), + ...(init.body === undefined + ? {} + : { 'content-type': 'application/json' }), + }, + ...(init.body === undefined ? {} : { body: JSON.stringify(init.body) }), + }); + } + + it('reads and moves the fence for an authenticated control plane', async () => { + // #given + const worker = makeWorker(); + const { env, ctx } = makeEnv(); + env.MAINTENANCE_ADMIN_SECRET = ADMIN_SECRET; + + // #then — a 0.19-era database reads as open with no row at all. + const initial = await worker.fetch( + fenceRequest({ method: 'GET' }), + env, + ctx, + ); + expect(initial.status).toBe(200); + expect(await initial.json()).toEqual({ state: 'open' }); + + // #when — the control plane drains, then locks. + const drained = await worker.fetch( + fenceRequest({ + method: 'POST', + body: { expected: 'open', next: 'draining' }, + }), + env, + ctx, + ); + expect(drained.status).toBe(200); + expect(await drained.json()).toEqual({ state: 'draining' }); + + // #then — a STALE expectation is a 409 carrying the current state, so the + // loser of a control-plane race can re-plan without a second round trip. + const stale = await worker.fetch( + fenceRequest({ + method: 'POST', + body: { expected: 'open', next: 'migration-locked' }, + }), + env, + ctx, + ); + expect(stale.status).toBe(409); + expect(await stale.json()).toMatchObject({ + reason: { code: 'FENCE_CAS_CONFLICT', state: 'draining' }, + }); + + // #then — entering proof-only names its key, and the read reports it. + await worker.fetch( + fenceRequest({ + method: 'POST', + body: { expected: 'draining', next: 'migration-locked' }, + }), + env, + ctx, + ); + const proof = await worker.fetch( + fenceRequest({ + method: 'POST', + body: { + expected: 'migration-locked', + next: 'proof-only', + proofKey: 'proof-1', + }, + }), + env, + ctx, + ); + expect(proof.status).toBe(200); + const observed = await worker.fetch( + fenceRequest({ method: 'GET' }), + env, + ctx, + ); + expect(await observed.json()).toEqual({ + state: 'proof-only', + proofKey: 'proof-1', + }); + }); + + it('rejects an unknown state name as a client error', async () => { + const worker = makeWorker(); + const { env, ctx } = makeEnv(); + env.MAINTENANCE_ADMIN_SECRET = ADMIN_SECRET; + + const response = await worker.fetch( + fenceRequest({ + method: 'POST', + body: { expected: 'open', next: 'quiesced' }, + }), + env, + ctx, + ); + + expect(response.status).toBe(400); + }); + + it("refuses 'proof-only' with no proof key", async () => { + const worker = makeWorker(); + const { env, ctx } = makeEnv(); + env.MAINTENANCE_ADMIN_SECRET = ADMIN_SECRET; + + const response = await worker.fetch( + fenceRequest({ + method: 'POST', + body: { expected: 'open', next: 'proof-only' }, + }), + env, + ctx, + ); + + expect(response.status).toBe(400); + }); + + it('401s a missing or wrong credential, and never leaks the state', async () => { + const worker = makeWorker(); + const { env, ctx } = makeEnv(); + env.MAINTENANCE_ADMIN_SECRET = ADMIN_SECRET; + + for (const token of [null, 'tok-ada']) { + const response = await worker.fetch( + fenceRequest({ method: 'GET', token }), + env, + ctx, + ); + expect(response.status).toBe(401); + expect(await response.json()).toEqual({ + error: 'authentication required', + }); + } + }); + + it('503s when the control-plane credential is unconfigured', async () => { + // #given — the fence is the control that stops a deployment executing, so + // an unauthenticated caller must never be able to move it. + const worker = makeWorker(); + const { env, ctx } = makeEnv(); + expect(env.MAINTENANCE_ADMIN_SECRET).toBeUndefined(); + + // #then + const response = await worker.fetch( + fenceRequest({ method: 'GET', token: null }), + env, + ctx, + ); + expect(response.status).toBe(503); + expect(await response.json()).toEqual({ + error: 'execution fence administration unavailable', + }); + }); + + it('503s when the control-plane credential equals the deployment identity secret', async () => { + const worker = makeWorker(); + const { env, ctx } = makeEnv(); + env.MAINTENANCE_ADMIN_SECRET = env.DEPLOYMENT_IDENTITY_SECRET; + + const response = await worker.fetch( + fenceRequest({ method: 'GET' }), + env, + ctx, + ); + expect(response.status).toBe(503); + }); + + it('405s a method the fence surface does not serve', async () => { + const worker = makeWorker(); + const { env, ctx } = makeEnv(); + env.MAINTENANCE_ADMIN_SECRET = ADMIN_SECRET; + + const response = await worker.fetch( + new Request('http://host/admin/execution-fence', { method: 'DELETE' }), + env, + ctx, + ); + expect(response.status).toBe(405); + }); +}); + +describe('createFlowsafeWorker drain inventory', () => { + const ADMIN_SECRET = 'maintenance-admin-secret-0000000002'; + + function inventoryRequest( + query = '', + init: { method?: string; token?: string | null } = {}, + ): Request { + const token = init.token === undefined ? ADMIN_SECRET : init.token; + return new Request(`http://host/admin/inventory${query}`, { + method: init.method ?? 'GET', + headers: token === null ? {} : { authorization: `Bearer ${token}` }, + }); + } + + it('serves the index, then one category, behind the shared admin gate', async () => { + // #given — a deployment with nothing in it yet: every inventory table is + // created lazily by the first feature that writes it, so this is the state + // a freshly provisioned Worker is really in. + const worker = makeWorker(); + const { env, ctx } = makeEnv(); + env.MAINTENANCE_ADMIN_SECRET = ADMIN_SECRET; + + // #when — no category names the index. + const index = await worker.fetch(inventoryRequest(), env, ctx); + + // #then — the categories, split by class, plus the rule an empty answer + // means something under. The contract travels with the answer because an + // operator who reads a count without it will lock too early. + expect(index.status).toBe(200); + const body = (await index.json()) as { + categories: Array<{ category: string; class: string }>; + unenumerable: Array<{ name: string }>; + drainProof: { reading: string; reachableFrom: string[] }; + }; + expect(body.categories.filter((c) => c.class === 'work')).toHaveLength(7); + expect(body.categories.filter((c) => c.class === 'standing')).toHaveLength( + 2, + ); + expect(body.unenumerable.map((entry) => entry.name)).toContain( + 'run-owner-recovery-journal', + ); + expect(body.drainProof.reachableFrom).toEqual(['draining']); + expect(body.drainProof.reading).toContain('point-in-time observation'); + + // #then — a category over a database with no tables answers EMPTY rather + // than faulting, and creates nothing to answer with. + const runs = await worker.fetch( + inventoryRequest('?category=runs'), + env, + ctx, + ); + expect(runs.status).toBe(200); + expect(await runs.json()).toEqual({ + category: 'runs', + class: 'work', + table: 'mastra_workflow_snapshot', + entries: [], + }); + }); + + it('refuses an unauthenticated read, and 503s when the credential is unconfigured', async () => { + // #given — this read enumerates every outstanding run, approval, and + // reservation on the deployment. There is no capability relay behind it, + // so an absent secret closes the surface rather than opening it. + const worker = makeWorker(); + const { env, ctx } = makeEnv(); + + // #then — unconfigured is 503, not 200. + expect((await worker.fetch(inventoryRequest(), env, ctx)).status).toBe(503); + + // #then — configured but unpresented is 401, and a wrong token is too: + // neither learns anything about the deployment's state. + env.MAINTENANCE_ADMIN_SECRET = ADMIN_SECRET; + expect( + (await worker.fetch(inventoryRequest('', { token: null }), env, ctx)) + .status, + ).toBe(401); + expect( + ( + await worker.fetch( + inventoryRequest('', { token: 'wrong-secret-0000000000000000000' }), + env, + ctx, + ) + ).status, + ).toBe(401); + }); + + it('rejects an unknown category, a malformed limit, and a malformed cursor', async () => { + // #given — a sweep is only a proof if it really reaches the end. A request + // the route quietly "fixed" would restart the scan, and an operator + // watching for two consecutive empty sweeps would wait forever. + const worker = makeWorker(); + const { env, ctx } = makeEnv(); + env.MAINTENANCE_ADMIN_SECRET = ADMIN_SECRET; + + for (const query of [ + '?category=not-a-category', + '?category=runs&limit=abc', + '?category=runs&limit=0', + '?category=runs&cursor=not-json', + '?category=runs&cursor=%5B%22one-part-only%22%5D', + ]) { + const response = await worker.fetch(inventoryRequest(query), env, ctx); + expect(response.status, query).toBe(400); + } + }); + + it('405s a method the inventory does not serve', async () => { + const worker = makeWorker(); + const { env, ctx } = makeEnv(); + env.MAINTENANCE_ADMIN_SECRET = ADMIN_SECRET; + + const response = await worker.fetch( + inventoryRequest('', { method: 'POST' }), + env, + ctx, + ); + expect(response.status).toBe(405); + }); +}); diff --git a/packages/flowsafe/src/host-kit/flowsafe-worker.ts b/packages/flowsafe/src/host-kit/flowsafe-worker.ts index 4131ee9..99ee332 100644 --- a/packages/flowsafe/src/host-kit/flowsafe-worker.ts +++ b/packages/flowsafe/src/host-kit/flowsafe-worker.ts @@ -40,27 +40,39 @@ import { credentialsMatch } from '../do-runner/deployment-identity.js'; import type { DurableObjectRunLifecycleHooks } from '../do-runner/durable-object.js'; import type { DeploymentIdentityDatabase, + ExecutionFenceStore, PurgeExpiredBackgroundTasksResult, RunArtifactPurger, RunDeadlineCursor, SnapshotDatabase, + StartIdempotencyStore, } from '../do-runner/index.js'; import { + assertExecutionFenceState, DEPLOYMENT_IDENTITY_HEADER, DeploymentIdentityError, + DeploymentInventory, + DoStatusError, deploymentIdentityHeaders, ensureDeploymentIdentityBindings, + executionFenceFor, + executionFenceReadingPayload, + InvalidInventoryRequestError, + isInventoryCategory, purgeExpiredBackgroundTasks, purgeExpiredNotifications, purgeExpiredScheduleTriggers, purgeExpiredThreadState, purgeExpiredThreads, purgeExpiredWorkflowRuns, + START_IDEMPOTENCY_TABLE, + startIdempotencyFor, sweepExpiredRunDeadlines, verifyDurableObjectDeploymentIdentity, verifyDurableObjectDeploymentRequest, } from '../do-runner/index.js'; import { validateTablePrefix } from '../do-runner/table-prefix.js'; +import { readBoundedBody } from '../http-body.js'; import { abandonApprovalsForRun, type ResumeRunFn } from './approval-bridge.js'; import { bearerActorAuthenticator } from './bearer-auth.js'; import { @@ -221,6 +233,17 @@ export interface FlowsafeWorkerEnv { APPROVAL_ALLOW_SELF_DECISION?: string; /** Maintenance purges terminal run snapshots older than this (default 30 days). */ RUN_RETENTION_DAYS?: string; + /** + * How long a spent idempotency key stays answerable after its run settled — + * the key-validity horizon (var; defaults to RUN_RETENTION_DAYS, and is + * floored at it). + * + * Set this ABOVE run retention when callers may retry a start later than this + * deployment keeps run summaries: until the horizon elapses such a retry is + * told ALREADY_SETTLED, and after it the same key reads as brand new and + * starts a second run. + */ + START_IDEMPOTENCY_RETENTION_DAYS?: string; /** Maintenance purges DECIDED approval records older than this (default 30 days). */ APPROVAL_RETENTION_DAYS?: string; /** @@ -461,6 +484,16 @@ interface ConfiguredApprovalServiceOptions { notify?: ApprovalNotificationSink; allowSelfDecision: SelfDecisionPolicy; stream?: ApprovalStreamSink; + /** + * REQUIRED, unlike its optional counterpart on HostApprovalServiceOptions: + * this interface is internal to the composer, both of its call sites are in + * this file, and every service the composer builds sits on a database whose + * fence it can name. Making it required is what keeps a third call site from + * being added later that silently builds an unfenced service — which would + * let a decision commit durably on a migration-locked deployment and then + * fail to resume. + */ + executionFence: ExecutionFenceStore; } function buildConfiguredApprovalService( @@ -485,6 +518,7 @@ function buildConfiguredApprovalService( notify: options.notify, allowSelfDecision: options.allowSelfDecision, stream: options.stream, + executionFence: options.executionFence, }); } @@ -516,6 +550,7 @@ export function createFlowsafeRunnerLifecycle( : undefined; const service = buildConfiguredApprovalService(config, env, topology, { store: approvalStoreFactoryFor(env.DB, storageTablePrefix).store(), + executionFence: executionFenceForEnv(env), waitUntil: options.waitUntil, notify: config.notify?.(env), allowSelfDecision, @@ -659,6 +694,120 @@ function json(payload: unknown, status = 200): Response { const MAINTENANCE_ADMIN_SECRET_PATTERN = /^[\x21-\x7e]{32,256}$/; const FLEET_SPEC_DIGEST_PATTERN = /^[a-f0-9]{64}$/; +/** + * What the shared admin gate decided. A union rather than `Response | null` + * because an authorized answer carries two things a caller needs: the + * credential the request presented, which the maintenance route forwards to its + * Durable Object on the delegating path below, and WHETHER this was that + * delegating path. Recovering the credential by re-reading the header would + * either re-duplicate the Bearer extraction this gate exists to own, or need an + * unreachable `undefined` branch to satisfy the types. + * + * `delegated` is reported rather than re-derived for the stronger reason: the + * rule that makes `MAINTENANCE_ADMIN_SECRET === undefined` mean "delegating" is + * enforced HERE — an absent secret refuses outright unless the caller asked to + * delegate — so a route re-testing the env var is restating a decision it + * cannot see, and would keep answering `true` if this gate's policy ever + * changed. One decision, reported once. + */ +type AdminCredentialDecision = + | { + readonly authorized: true; + readonly credential: string; + /** + * The credential is a downstream-verified capability token relayed by + * this Worker, not a match against MAINTENANCE_ADMIN_SECRET. + */ + readonly delegated: boolean; + } + | { readonly authorized: false; readonly response: Response }; + +/** + * The credential preamble EVERY /admin surface runs before it does anything. + * + * One function rather than a copy per route because this is the trust boundary + * itself (docs/security-threat-model.md, "The provisioning boundary"): it + * proves MAINTENANCE_ADMIN_SECRET is configured, proves it is DISTINCT from the + * deployment identity secret (sharing them would let a Worker-to-DO credential + * move the fence, and vice versa), and constant-time compares the request's + * Bearer token against it. A second copy is a second place for one of those + * three to be dropped in a hurry, and the inventory route lands here next. + * + * The surfaces differ in exactly ONE thing, which is why it is a parameter + * rather than a fork: what an ABSENT secret means. `/admin/execution-fence` + * always refuses — the fence is the control that stops a deployment executing, + * so an unauthenticated caller must never reach it. The maintenance routes + * delegate instead when the fleet requires capability tokens, because there the + * Durable Object verifies a signed capability and this Worker is only a relay; + * the longer credential cap applies to that path alone, since a capability + * token is not a shared secret. + */ +async function authorizeAdminCredential( + request: Request, + env: Env, + options: { + /** Names the surface in the config-error log and the 503 body. */ + readonly surface: string; + /** + * Whether an absent MAINTENANCE_ADMIN_SECRET delegates authentication + * downstream rather than refusing. The caller folds its own policy into + * this boolean so the gate stays about credentials only. + */ + readonly delegateWhenUnconfigured: boolean; + }, +): Promise { + const { surface } = options; + const unavailable = (reason: string): AdminCredentialDecision => { + console.error( + JSON.stringify({ + type: 'config-error', + var: 'MAINTENANCE_ADMIN_SECRET', + reason, + }), + ); + return { + authorized: false, + response: json({ error: `${surface} unavailable` }, 503), + }; + }; + const unauthenticated = (): AdminCredentialDecision => ({ + authorized: false, + response: json({ error: 'authentication required' }, 401), + }); + const expected = env.MAINTENANCE_ADMIN_SECRET; + const delegating = expected === undefined && options.delegateWhenUnconfigured; + if (!delegating) { + if ( + expected === undefined || + !MAINTENANCE_ADMIN_SECRET_PATTERN.test(expected) + ) { + return unavailable(`${surface} is not configured`); + } + if (await credentialsMatch(expected, env.DEPLOYMENT_IDENTITY_SECRET)) { + return unavailable( + 'maintenance and deployment identity credentials must differ', + ); + } + } + const credential = bearerCredential(request); + const maximumCredentialLength = delegating ? 2_048 : 256; + if (!credential || credential.length > maximumCredentialLength) { + return unauthenticated(); + } + if ( + expected !== undefined && + !(await credentialsMatch(credential, expected)) + ) { + return unauthenticated(); + } + return { authorized: true, credential, delegated: delegating }; +} + +/** The Bearer credential a request presents, if it presents a well-formed one. */ +function bearerCredential(request: Request): string | undefined { + return request.headers.get('authorization')?.match(/^Bearer\s+(.+)$/i)?.[1]; +} + async function maintenanceAdminResponse( request: Request, env: Env, @@ -675,56 +824,14 @@ async function maintenanceAdminResponse( if (request.method !== expectedMethod) { return json({ error: 'method not allowed' }, 405); } - const expected = env.MAINTENANCE_ADMIN_SECRET; - if ( - expected === undefined && - env.FLEET_MAINTENANCE_CAPABILITIES !== 'required' - ) { - console.error( - JSON.stringify({ - type: 'config-error', - var: 'MAINTENANCE_ADMIN_SECRET', - reason: 'maintenance administration is not configured', - }), - ); - return json({ error: 'maintenance administration unavailable' }, 503); - } - if ( - expected !== undefined && - !MAINTENANCE_ADMIN_SECRET_PATTERN.test(expected) - ) { - console.error( - JSON.stringify({ - type: 'config-error', - var: 'MAINTENANCE_ADMIN_SECRET', - reason: 'maintenance administration is not configured', - }), - ); - return json({ error: 'maintenance administration unavailable' }, 503); - } - if ( - expected !== undefined && - (await credentialsMatch(expected, env.DEPLOYMENT_IDENTITY_SECRET)) - ) { - console.error( - JSON.stringify({ - type: 'config-error', - var: 'MAINTENANCE_ADMIN_SECRET', - reason: 'maintenance and deployment identity credentials must differ', - }), - ); - return json({ error: 'maintenance administration unavailable' }, 503); - } - const actual = request.headers - .get('authorization') - ?.match(/^Bearer\s+(.+)$/i)?.[1]; - const maximumCredentialLength = expected === undefined ? 2_048 : 256; - if (!actual || actual.length > maximumCredentialLength) { - return json({ error: 'authentication required' }, 401); - } - if (expected !== undefined && !(await credentialsMatch(actual, expected))) { - return json({ error: 'authentication required' }, 401); - } + const gate = await authorizeAdminCredential(request, env, { + surface: 'maintenance administration', + // An unconfigured secret is survivable HERE and only here: a fleet that + // requires capability tokens authenticates at the maintenance DO, which + // verifies a signed capability this Worker only relays. + delegateWhenUnconfigured: env.FLEET_MAINTENANCE_CAPABILITIES === 'required', + }); + if (!gate.authorized) return gate.response; const deploymentSpecDigest = env.FLEET_SPEC_DIGEST; if ( deploymentSpecDigest !== undefined && @@ -744,13 +851,19 @@ async function maintenanceAdminResponse( `http://maintenance/${operation}`, { method: expectedMethod, - headers: - expected === undefined - ? { authorization: `Bearer ${actual}` } - : deploymentIdentityHeaders(env.DEPLOYMENT_IDENTITY_SECRET), + // The gate's own verdict, not a second reading of the env var it decided + // on: a delegated request carries a capability the maintenance DO + // verifies, so this Worker relays the credential unchanged; anything else + // was authenticated HERE and travels on the deployment identity. + headers: gate.delegated + ? { authorization: `Bearer ${gate.credential}` } + : deploymentIdentityHeaders(env.DEPLOYMENT_IDENTITY_SECRET), }, ); - if (expected === undefined) return response; + // Nothing to enrich on the delegated path: the digest check below belongs to + // the deployment this Worker authenticated for, and a relayed response is the + // maintenance DO's own answer. + if (gate.delegated) return response; if (!response.ok || deploymentSpecDigest === undefined) return response; const payload: unknown = await response.json(); if (!payload || typeof payload !== 'object' || Array.isArray(payload)) { @@ -762,6 +875,242 @@ async function maintenanceAdminResponse( }); } +/** + * This composer's env-shaped view of the ONE per-database fence memo + * (do-runner/execution-fence.ts). Every call site here holds an `env` rather + * than a binding, so the unwrapping happens once instead of at each of them; + * the identity guarantee — admin route, approval service, and runner runtime + * all on the same store for the same database — belongs to the shared memo. + */ +function executionFenceForEnv( + env: Env, +): ExecutionFenceStore { + return executionFenceFor(env.DB); +} + +/** + * The same env-shaped view of the ONE per-database start-reservation memo + * (do-runner/start-idempotency.ts), for the same reason: the run router + * reserves here, and the runtime inside the run object settles there, and both + * must be the store built from THIS env's binding. + */ +function startIdempotencyForEnv( + env: Env, +): StartIdempotencyStore { + return startIdempotencyFor(env.DB); +} + +const EXECUTION_FENCE_ADMIN_PATH = '/admin/execution-fence'; +const EXECUTION_FENCE_ADMIN_MAX_BODY_BYTES = 4_096; + +/** + * The deployment execution fence's control-plane surface, behind the SAME gate + * `maintenanceAdminResponse` uses (`authorizeAdminCredential`) — it is the same + * trust boundary (the provisioning TCB, docs/security-threat-model.md), and + * splitting the credential would only add a second secret an operator can get + * wrong. + * + * `GET` reports the state; `POST` moves it, compare-and-set on `expected`. + * Transition POLICY is the HOST's: this package enforces only the state + * vocabulary, the CAS, and the rule that entering 'proof-only' names a proof + * key. Which sequence of states a migration walks is the control plane's + * business, and hard-coding it here would freeze an operational procedure into + * a published package. + * + * Answers on a mis-provisioned deployment are NOT reachable: the identity gate + * 503s before this dispatch. Accepted — an operator whose bindings are wrong + * has a bigger problem than the fence, and relaxing the identity gate to serve + * this route would put an unverified database behind a control-plane write. + */ +async function executionFenceAdminResponse( + request: Request, + env: Env, +): Promise { + const url = new URL(request.url); + if (url.pathname !== EXECUTION_FENCE_ADMIN_PATH) return null; + if (request.method !== 'GET' && request.method !== 'POST') { + return json({ error: 'method not allowed' }, 405); + } + const gate = await authorizeAdminCredential(request, env, { + surface: 'execution fence administration', + // Unconfigured is 503, never open: the fence is the control that stops a + // deployment executing, so an unauthenticated caller must never move it. + // There is no capability-token relay behind this route to delegate to. + delegateWhenUnconfigured: false, + }); + if (!gate.authorized) return gate.response; + const fence = executionFenceForEnv(env); + try { + if (request.method === 'GET') { + return json(executionFenceReadingPayload(await fence.read())); + } + const raw = await readBoundedBody( + request, + EXECUTION_FENCE_ADMIN_MAX_BODY_BYTES, + 'execution fence body exceeds limit', + ); + if (!raw.ok) { + return json( + { error: 'a JSON object body is required' }, + raw.reason === 'payload-too-large' ? 413 : 400, + ); + } + let parsed: unknown; + try { + parsed = raw.text === '' ? undefined : JSON.parse(raw.text); + } catch { + parsed = undefined; + } + if ( + parsed === null || + typeof parsed !== 'object' || + Array.isArray(parsed) + ) { + return json({ error: 'a JSON object body is required' }, 400); + } + const body = parsed as { + expected?: unknown; + next?: unknown; + proofKey?: unknown; + }; + const reading = await fence.transition({ + expected: assertExecutionFenceState(body.expected, 'expected'), + next: assertExecutionFenceState(body.next, 'next'), + ...(body.proofKey === undefined ? {} : { proofKey: body.proofKey }), + }); + return json({ state: reading.state }); + } catch (error) { + if (error instanceof DoStatusError) { + return json( + { + error: error.message, + ...(error.reason === undefined ? {} : { reason: error.reason }), + }, + error.status, + ); + } + console.error( + JSON.stringify({ + type: 'execution-fence-admin-error', + reason: error instanceof Error ? error.message : String(error), + }), + ); + return json({ error: 'execution fence administration failed' }, 500); + } +} + +const INVENTORY_ADMIN_PATH = '/admin/inventory'; + +/** + * The deployment's drain-proof surface uses the same credential helper as the + * fence and maintenance routes. Their absent-secret policies differ: + * maintenance routes may delegate to a fleet capability; fence and inventory + * routes never do. + * + * WHAT IT IS FOR. An operator holds the fence in `draining` and needs to prove, + * with no side effects, that this deployment holds no executable or resumable + * work before locking it for a migration. Without such a proof the only + * available answer is a wait-and-hope, and a suspended run that survives it + * gets resumed by two deployments at once. + * + * HOW TO READ THE ANSWER — the contract the index response also states, in + * full, because this is the part a hurried operator skips: + * + * Each result is a point-in-time observation, not a snapshot. Rows can enter + * or leave `work` categories while draining admits work. An empty result + * cannot over-count work, and keyset pagination never skips a row that + * existed before the sweep started. + * + * The proof is TWO consecutive full sweeps, at least one 60-second + * RUN_OWNER_RECOVERY_DELAY_MS cadence apart, in which every `work` category + * comes back empty. One sweep cannot cover the window in which a run object + * has journalled its recovery key but not yet written its D1 owner row; two, + * spaced that far, can. + * + * An empty sweep from `draining` contributes to the two-sweep proof. After + * the transition to `migration-locked`, an empty post-lock re-sweep is + * conclusive because the lock refuses new work. A non-empty one means work is + * still outstanding, either because it entered after the proof or because + * the lock parked it before it finished. Return to `draining` and repeat the + * proof. + * + * `standing` categories (schedules, provider subscriptions) are reported for + * reconciliation and are never required to be empty: they are configuration a + * migration carries across. + * + * GET only. Every query underneath is a bare SELECT that never creates schema, + * so running this against a deployment about to be copied does not change what + * is being copied. + */ +async function inventoryAdminResponse( + request: Request, + env: Env, + storageTablePrefix: string | undefined, +): Promise { + const url = new URL(request.url); + if (url.pathname !== INVENTORY_ADMIN_PATH) return null; + if (request.method !== 'GET') { + return json({ error: 'method not allowed' }, 405); + } + const gate = await authorizeAdminCredential(request, env, { + surface: 'deployment inventory', + // Unconfigured is 503, never open: this read enumerates every outstanding + // run, approval, and reservation on the deployment. There is no capability + // relay behind it to delegate to, either. + delegateWhenUnconfigured: false, + }); + if (!gate.authorized) return gate.response; + try { + const inventory = new DeploymentInventory(env.DB, { + ...(storageTablePrefix === undefined + ? {} + : { tablePrefix: storageTablePrefix }), + }); + const category = url.searchParams.get('category'); + if (category === null || category === '') { + return json(inventory.index()); + } + if (!isInventoryCategory(category)) { + throw new InvalidInventoryRequestError( + `unknown inventory category '${category}'`, + ); + } + const rawLimit = url.searchParams.get('limit'); + // Parsed here rather than coerced inside the store: `Number('abc')` is NaN + // and `Number('')` is 0, and both would otherwise reach the clamp as a + // number that means something the caller never asked for. + if (rawLimit !== null && !/^[0-9]{1,4}$/.test(rawLimit)) { + throw new InvalidInventoryRequestError( + 'inventory limit must be a positive integer', + ); + } + const cursor = url.searchParams.get('cursor'); + return json( + await inventory.read(category, { + ...(cursor === null ? {} : { cursor }), + ...(rawLimit === null ? {} : { limit: Number(rawLimit) }), + }), + ); + } catch (error) { + if (error instanceof DoStatusError) { + return json( + { + error: error.message, + ...(error.reason === undefined ? {} : { reason: error.reason }), + }, + error.status, + ); + } + console.error( + JSON.stringify({ + type: 'inventory-admin-error', + reason: error instanceof Error ? error.message : String(error), + }), + ); + return json({ error: 'deployment inventory failed' }, 500); + } +} + export function createFlowsafeWorker( config: FlowsafeWorkerConfig, ): FlowsafeWorker { @@ -785,6 +1134,7 @@ export function createFlowsafeWorker( buildService: (store) => buildConfiguredApprovalService(config, env, topology, { store, + executionFence: executionFenceForEnv(env), waitUntil, notify, stream, @@ -866,6 +1216,28 @@ export function createFlowsafeWorker( // cannot leave unbounded run-ownership tombstones or expose a live row // without its authorization record. resourceOwnerTable: RESOURCE_OWNERSHIP_TABLE, + // Start reservations join the same transaction, for the same reason: + // an idempotency key must never be reaped while the run it names is + // still readable, or the next retry of that key would start a second + // run beside the live one. The horizon defaults to run retention — + // START_IDEMPOTENCY_RETENTION_DAYS is what a host sets when its callers + // retry for longer than it keeps run summaries. + startIdempotencyTable: START_IDEMPOTENCY_TABLE, + ...(env.START_IDEMPOTENCY_RETENTION_DAYS === undefined + ? {} + : { + startIdempotencyTtlMs: + numberVar( + env.START_IDEMPOTENCY_RETENTION_DAYS, + 30, + 'START_IDEMPOTENCY_RETENTION_DAYS', + { allowZero: true }, + ) * + 24 * + 60 * + 60 * + 1000, + }), }); } catch (error) { recordFailure('retention-purge', error); @@ -911,7 +1283,7 @@ export function createFlowsafeWorker( recordFailure('thread-retention-purge', error); } } - // Background-task TTL cleanup (Track B, opt-in). Its OWN try/catch, like + // Background-task TTL cleanup (opt-in). Its OWN try/catch, like // every sibling duty: a wedged background-task purge must cost the // run-snapshot, approval, and thread purges nothing. let backgroundTasksPurged: PurgeExpiredBackgroundTasksResult | undefined; @@ -926,7 +1298,7 @@ export function createFlowsafeWorker( recordFailure('background-task-purge', error); } } - // Track C notification TTL (opt-in). Its OWN try/catch, like every sibling + // Notification TTL cleanup (opt-in). Its OWN try/catch, like every sibling // duty. optionalNumberVar (GATES the duty; unset/garbage => do not delete). let notificationsPurged: number | undefined; const notificationRetentionDays = optionalNumberVar( @@ -944,7 +1316,7 @@ export function createFlowsafeWorker( recordFailure('notification-purge', error); } } - // Track C thread-state TTL (opt-in). Same isolation + opt-in posture. + // Thread-state TTL cleanup (opt-in). Same isolation + opt-in posture. let threadStatePurged: number | undefined; const threadStateRetentionDays = optionalNumberVar( env.THREAD_STATE_RETENTION_DAYS, @@ -961,7 +1333,7 @@ export function createFlowsafeWorker( recordFailure('thread-state-purge', error); } } - // Track D schedule-trigger history TTL (opt-in). Same isolation + opt-in + // Schedule-trigger history TTL cleanup (opt-in). Same isolation + opt-in // posture. Only the fire HISTORY expires; schedule config rows are reaped // only at deployment teardown. let scheduleTriggersPurged: number | undefined; @@ -1009,7 +1381,7 @@ export function createFlowsafeWorker( : { ok: false, error: failures.join('; ') }; } - // Track D schedule tick — its OWN failure-isolated duty (own try/catch, own log + // Schedule tick — its OWN failure-isolated duty (own try/catch, own log // line). A wedged fire pass must cost the other duties nothing, and vice // versa (the same failure-isolation rationale). async function runScheduleTickDuty( @@ -1119,6 +1491,23 @@ export function createFlowsafeWorker( const maintenanceAdmin = await maintenanceAdminResponse(request, env); if (maintenanceAdmin) return maintenanceAdmin; + // The execution fence's control plane, beside the maintenance admin + // routes and ahead of every tenant router: it must answer while the + // deployment is refusing tenant traffic, which is the whole state it + // exists to report and clear. + const fenceAdmin = await executionFenceAdminResponse(request, env); + if (fenceAdmin) return fenceAdmin; + + // Beside the fence for the same reason: the drain proof is read WHILE + // the deployment is refusing tenant traffic, so it must not sit behind + // a router that a drain is trying to empty. + const inventoryAdmin = await inventoryAdminResponse( + request, + env, + storageTablePrefix, + ); + if (inventoryAdmin) return inventoryAdmin; + const waitUntil = (promise: Promise): void => ctx.waitUntil(promise); const topology = createDoRunTopology( @@ -1128,7 +1517,7 @@ export function createFlowsafeWorker( const notify = config.notify?.(env); const selfDecision = parseSelfDecision(env); // Fetch-scope live fan-out sink: present iff a hub is bound (streaming is - // opt-in, DL-019). Each publish rides ctx.waitUntil (DL-020) and is + // opt-in). Each publish rides ctx.waitUntil and is // contained — a failed fan-out logs and never fails the mutation. const hub = env.HUB; let streamSink: ApprovalStreamSink | undefined; @@ -1173,7 +1562,7 @@ export function createFlowsafeWorker( if (agentResponse) return agentResponse; } - // Optional stream stage (DL-015/DL-019): mounted only when BOTH the hub + // Optional stream stage: mounted only when BOTH the hub // binding and the ticket secret are present. Every route is under // /api/stream/, so it composes ahead of the approval router without // touching the /api/* run_worker_first entry. @@ -1189,29 +1578,29 @@ export function createFlowsafeWorker( if (streamResponse) return streamResponse; } - // Optional Track C signal stage (P6): the host-built createSignalRouter, - // closed over THIS request's resolver. `/api/threads/*` — composes ahead - // of approvals/runs without overlap. Absent seam => unmounted. + // Optional signal stage: the host-built createSignalRouter closes over + // THIS request's resolver. `/api/threads/*` composes ahead of + // approvals/runs without overlap. Absent seam => unmounted. const signalRouter = config.buildSignalRouter?.(resolve, env); if (signalRouter) { const signalResponse = await signalRouter(request); if (signalResponse) return signalResponse; } - // Optional Track F goal stage (P6-lite, DL-018): the host-built - // createObjectiveRouter, closed over THIS request's resolver. - // `/api/threads/:threadId/goal` — composes after signals (non-overlapping) - // and ahead of approvals/runs. Absent seam ⇒ unmounted, byte-identical. + // Optional goal stage: the host-built createObjectiveRouter closes over + // THIS request's resolver. `/api/threads/:threadId/goal` composes after + // signals and ahead of approvals/runs. Absent seam => unmounted and + // byte-identical. const objectiveRouter = config.buildObjectiveRouter?.(resolve, env); if (objectiveRouter) { const objectiveResponse = await objectiveRouter(request); if (objectiveResponse) return objectiveResponse; } - // Optional Track D schedule CRUD stage (DL-013): the host-built - // createScheduleRouter, closed over THIS request's resolver. `/api/schedules/*` - // — composes after goals (non-overlapping), ahead of approvals/runs. Absent - // seam ⇒ unmounted, byte-identical. + // Optional schedule CRUD stage: the host-built createScheduleRouter + // closes over THIS request's resolver. `/api/schedules/*` composes + // after goals and ahead of approvals/runs. Absent seam => unmounted + // and byte-identical. const scheduleRouter = config.buildScheduleRouter?.(resolve, env); if (scheduleRouter) { const scheduleResponse = await scheduleRouter(request); @@ -1233,6 +1622,16 @@ export function createFlowsafeWorker( status: topology.status, resume: topology.resume, terminate: topology.terminate, + // Both halves come from this env's own bindings: the store from the + // per-database memo (so the runtime inside the run object settles the + // very rows this router reserved), and the probe from the same DO + // topology every other run operation travels through (so "is it + // live?" is asked of the one object that could be running it). + startIdempotency: { + store: startIdempotencyForEnv(env), + live: topology.startLiveness, + executionFence: executionFenceForEnv(env), + }, beforeStart: beforeStart ? (context, workflowId, inputData) => beforeStart(context, env, workflowId, inputData) @@ -1241,8 +1640,8 @@ export function createFlowsafeWorker( ? (context, workflowId, runId, body) => beforeResume(context, env, workflowId, runId, body) : undefined, - // D4 self-healing, waitUntil-detached — the shared wrapper owns the - // detach + reconcile-error logging. + // Self-healing approval reconciliation is waitUntil-detached; the + // shared wrapper owns the detach + reconcile-error logging. reconcileApprovals: reconcileApprovalsOnStatusDetached( config.systemPrincipalId, waitUntil, diff --git a/packages/flowsafe/src/host-kit/host-approval-service.test.ts b/packages/flowsafe/src/host-kit/host-approval-service.test.ts index 30c9425..5c45c74 100644 --- a/packages/flowsafe/src/host-kit/host-approval-service.test.ts +++ b/packages/flowsafe/src/host-kit/host-approval-service.test.ts @@ -94,6 +94,8 @@ describe('buildHostApprovalService allowSelfDecision passthrough', () => { // non-'suspended' summary means resumeRunWithRequeue queues nothing. resumeRun: async (record) => ({ runId: record.runId, status: 'success' }), allowSelfDecision, + // In-memory approval store — no database, nothing to fence. + executionFence: 'none', }); } diff --git a/packages/flowsafe/src/host-kit/host-approval-service.ts b/packages/flowsafe/src/host-kit/host-approval-service.ts index e3d0731..1444461 100644 --- a/packages/flowsafe/src/host-kit/host-approval-service.ts +++ b/packages/flowsafe/src/host-kit/host-approval-service.ts @@ -29,6 +29,7 @@ import { trustAutomationPrincipal, } from '../approval-api/index.js'; import { type AuditQueue, queueAuditSink } from '../audit-export/index.js'; +import type { ExecutionFenceWiring } from '../do-runner/execution-fence.js'; import { validateTablePrefix } from '../do-runner/table-prefix.js'; import { type ResumeRunFn, @@ -173,6 +174,25 @@ export interface HostApprovalServiceOptions { * at fetch scope. Undefined means no live fan-out (a poll-only host). */ stream?: ApprovalStreamSink; + /** + * The deployment execution fence (do-runner/execution-fence.ts), forwarded to + * ApprovalServiceOptions.executionFence, or `'none'` for a service with no + * database behind it. + * + * REQUIRED, and the `'none'` branch is genuinely dangerous rather than merely + * unusual: `'none'` makes decide() unfenced, and decide COMMITS the decision + * and only then resumes. A migration-locked deployment would durably record a + * decision — with its audit trail and its notification — whose resume then + * 503s, and the deployment taking over inherits a decided approval with + * nothing behind it. Write it only for a service that has no database to + * fence against at all. + * + * This function receives an ApprovalStore rather than a database, so it + * cannot build the store itself the way `init({ DB })` can — which is exactly + * why the option is required rather than optional: the host is the only place + * the wiring can happen, so the type has to make it name one. + */ + executionFence: ExecutionFenceWiring; } /** @@ -202,6 +222,12 @@ export function buildHostApprovalService( notify: options.notify, stream: options.stream, allowSelfDecision: options.allowSelfDecision, + // Forwarded as written, opt-out included: ApprovalService now requires the + // same wiring this composer does, so there is nothing left to resolve here + // — and resolving `'none'` to `undefined` on the way down would have + // erased, one layer above the gate, the distinction between a host that + // named the opt-out and one that never held a fence at all. + executionFence: options.executionFence, resumeRun: resumeRunWithRequeue( options.resumeRun, () => service, diff --git a/packages/flowsafe/src/host-kit/index.ts b/packages/flowsafe/src/host-kit/index.ts index 38ff31a..137e6d9 100644 --- a/packages/flowsafe/src/host-kit/index.ts +++ b/packages/flowsafe/src/host-kit/index.ts @@ -113,10 +113,14 @@ export { export type { WorkflowIdSource } from './registration.js'; export { assertWorkflowsRegistered } from './registration.js'; export { requireResourceAccess } from './resource-access.js'; -export { RunRouteError } from './run-route-error.js'; +export { RunRouteError, runRouteReason } from './run-route-error.js'; export type { RunRouter, RunRouterOptions, + // The type of RunRouterOptions.startIdempotency, published because that + // option is required: a host has to be able to name the shape it is being + // asked to supply. + RunRouterStartIdempotency, RunStartInput, } from './run-router.js'; export { createRunRouter } from './run-router.js'; diff --git a/packages/flowsafe/src/host-kit/run-route-error.ts b/packages/flowsafe/src/host-kit/run-route-error.ts index 6c099c0..d634e49 100644 --- a/packages/flowsafe/src/host-kit/run-route-error.ts +++ b/packages/flowsafe/src/host-kit/run-route-error.ts @@ -24,3 +24,29 @@ export class RunRouteError extends Error { this.reason = reason; } } + +/** + * The Durable Object's own structured refusal, if this error carries one. + * + * The taxonomy's rule is that a `reason` is a SCREAMING_SNAKE code the DO + * deliberately published (do-error-response.ts), and every router that fronts a + * DO must pass one through with its status intact. Without this, a router that + * collapses 5xx to a bare 500 — the shape agent-host/router.ts and + * stream-router.ts both had — turns "this deployment is fenced, retry after the + * migration" into "I am broken", and the caller cannot tell a retryable + * operational state from a code fault. + * + * Narrow on purpose: only a plain object with a string `code` qualifies, so an + * upstream body that happened to carry a `reason` field of some other shape + * cannot widen what a 5xx surfaces. + */ +export function runRouteReason( + error: RunRouteError, +): { code: string } | undefined { + const { reason } = error; + if (typeof reason !== 'object' || reason === null || Array.isArray(reason)) { + return undefined; + } + const { code } = reason as { code?: unknown }; + return typeof code === 'string' ? (reason as { code: string }) : undefined; +} diff --git a/packages/flowsafe/src/host-kit/run-router.test.ts b/packages/flowsafe/src/host-kit/run-router.test.ts index 3e1d15a..0955bb5 100644 --- a/packages/flowsafe/src/host-kit/run-router.test.ts +++ b/packages/flowsafe/src/host-kit/run-router.test.ts @@ -3,7 +3,7 @@ // (401 -> coarse RUN_START_ROLES -> per-workflow allowedRoles), the catalog, the // start/status/resume/terminate routes and their error mapping, the suspension bridge's // attribution (the starting actor becomes requestedBy, so they cannot decide -// their own run), and the D4 reconcileApprovals self-healing hook on status +// their own run), and the reconcileApprovals self-healing hook on status // reads. // // Driven with real InMemoryApprovalStore + ApprovalService (no mocks) and @@ -12,6 +12,7 @@ import { describe, expect, it, vi } from 'vitest'; +import { openSqlite, sqliteUnitDatabase } from '../../test-support/sqlite.js'; import { type ApprovalActor, ApprovalService, @@ -21,11 +22,15 @@ import { type SelfDecisionPolicy, } from '../approval-api/index.js'; import { + ExecutionFencedError, + type ExecutionFenceWiring, InvalidRunRequestError, RunLifecycleBlockedError, RunNotSuspendedError, type RunSummary, RunTerminalConflictError, + type StartIdempotencyDatabase, + StartIdempotencyStore, UnknownRunError, } from '../do-runner/index.js'; import { reconcileApprovalsOnStatus } from './approval-bridge.js'; @@ -86,11 +91,17 @@ interface HarnessOptions { resume?: RunRouterOptions['resume']; terminate?: NonNullable; reconcileApprovals?: RunRouterOptions['reconcileApprovals']; - // F9: feeds the resolver's allowSelfDecision now (the run-router no longer + // Feed the resolver's allowSelfDecision now (the run-router no longer // owns a selfDecision knob), driving the catalog's canSelfDecide echo. selfDecision?: SelfDecisionPolicy; resourceOwner?: ReqActor; approvalCreateFailures?: number; + /** + * How this harness honours `idempotencyKey`. Defaults to the typed opt-out, + * which is what every earlier unkeyed-start case in this file wants: unkeyed + * starts are unaffected; keyed starts on an unwired host refuse. + */ + startIdempotency?: RunRouterOptions['startIdempotency']; } function makeHarness(options: HarnessOptions = {}) { @@ -120,7 +131,9 @@ function makeHarness(options: HarnessOptions = {}) { }, }) as ApprovalStore; // The bridge queues records through the request-scoped deployment service. - const service = new ApprovalService({ store }); + // In-memory store, no database to fence against: the opt-out is written down + // rather than defaulted — see ExecutionFenceWiring. + const service = new ApprovalService({ store, executionFence: 'none' }); const started: Array<{ workflowId: string; runId: string; @@ -137,9 +150,10 @@ function makeHarness(options: HarnessOptions = {}) { : undefined; }, storeFactory: backend, - buildService: () => new ApprovalService({ store: requestStore }), + buildService: () => + new ApprovalService({ store: requestStore, executionFence: 'none' }), newRunId: () => 'generated-run-id', - // F9: the SoD exemption policy now feeds the resolver, so the catalog echo + // The SoD exemption policy now feeds the resolver, so the catalog echo // reads context.canSelfDecide (the run-router no longer takes its own knob). allowSelfDecision: options.selfDecision, }); @@ -147,6 +161,7 @@ function makeHarness(options: HarnessOptions = {}) { workflows: WORKFLOWS, resolve, systemPrincipalId: SYSTEM.id, + startIdempotency: options.startIdempotency ?? 'none', start: async (input) => { await backend.resources().claim('run', input.runId, { kind: input.principal.kind, @@ -529,7 +544,7 @@ describe('createRunRouter — POST /runs', () => { ).toBe(404); }); - it('400s a client-pinned runId and mints an opaque id itself (INV-1)', async () => { + it('400s a client-pinned runId and mints an opaque id itself', async () => { // #given — a client may never choose the addressing id const { handle, started } = makeHarness(); @@ -905,7 +920,7 @@ describe('createRunRouter — GET status and POST resume', () => { }); }); -describe('createRunRouter — reconcileApprovals hook (D4 self-healing)', () => { +describe('createRunRouter — reconcileApprovals self-healing hook', () => { it('invokes reconcileApprovals when a status read reports the run suspended', async () => { // #given const calls: Array<{ workflowId: string; runId: string }> = []; @@ -1048,3 +1063,450 @@ describe('createRunRouter — error mapping', () => { expect(await response?.json()).toEqual({ error: 'd1 exploded' }); }); }); + +// --------------------------------------------------------------------------- +// Owner-bound idempotent start, on the workflow surface. +// +// Every case below is about a PAID first step: the assertions count +// EXECUTIONS, not responses, because a router that answers correctly while +// starting a second run has failed at exactly the thing the key was bought for. +// --------------------------------------------------------------------------- + +/** A run router whose reservations live in a real table over node:sqlite. */ +function keyedHarness( + options: HarnessOptions & { + live?: (workflowId: string, runId: string) => Promise; + executionFence?: ExecutionFenceWiring; + now?: () => number; + } = {}, +) { + const sqlite = openSqlite(); + const store = new StartIdempotencyStore( + sqliteUnitDatabase(sqlite) as StartIdempotencyDatabase, + options.now ? { now: options.now } : {}, + ); + const persisted = new Map(); + const executions: string[] = []; + // The stand-in for a run object's in-flight set: what a liveness probe + // actually answers from. Populated for the whole of a start, so a sibling + // request that arrives mid-execution sees the truth a DO would report. + const inFlight = new Set(); + let runSeq = 0; + const harness = makeHarness({ + ...options, + startIdempotency: { + store, + live: options.live ?? (async (_workflowId, runId) => inFlight.has(runId)), + executionFence: options.executionFence ?? 'none', + }, + start: + options.start ?? + (async ({ runId, workflowId }) => { + inFlight.add(runId); + try { + // The stand-in for the paid first step. The awaited tick is what lets + // a parallel sibling actually interleave here. + await Promise.resolve(); + executions.push(runId); + const summary: RunSummary = { + runId, + status: 'success', + result: { ok: true, workflowId }, + }; + persisted.set(runId, summary); + return summary; + } finally { + inFlight.delete(runId); + } + }), + status: + options.status ?? (async (_workflowId, runId) => persisted.get(runId)), + }); + return { + ...harness, + store, + persisted, + executions, + /** Distinct ids per mint, so a converged retry is visible as convergence. */ + nextRunId: () => { + runSeq += 1; + return `mint_${runSeq}`; + }, + }; +} + +describe('createRunRouter — idempotent start', () => { + it('refuses a key on a host that wired no reservation store', async () => { + // #given the typed opt-out — the default in this file + const { handle } = makeHarness(); + + // #when + const response = await handle( + req('/runs', { + body: { workflowId: 'open-flow', idempotencyKey: 'key-1' }, + }), + ); + + // #then 503 with a code, never a silent unkeyed start: ignoring the key + // would answer an exactly-once request with at-least-once behaviour. + expect(response?.status).toBe(503); + expect(await response?.json()).toMatchObject({ + reason: { code: 'IDEMPOTENT_START_UNSUPPORTED' }, + }); + }); + + it('keeps refusing a caller-supplied runId even alongside a key', async () => { + // #given — the key is the one identifier a client may choose; the runId + // names a slot in every store this deployment has and stays server-minted. + const { handle } = keyedHarness(); + + // #when + const response = await handle( + req('/runs', { + body: { + workflowId: 'open-flow', + runId: 'chosen', + idempotencyKey: 'key-1', + }, + }), + ); + + // #then the pinned 400 is untouched by owner-bound idempotent start + expect(response?.status).toBe(400); + expect(await response?.json()).toEqual({ + error: 'runId is server-assigned', + }); + }); + + it('starts once and replays the SAME run for a retry, with no second execution', async () => { + // #given a host whose first step is expensive + const { handle, executions } = keyedHarness(); + + // #when the response to the first call is lost and the client retries + const first = await handle( + req('/runs', { + body: { workflowId: 'open-flow', idempotencyKey: 'key-1' }, + }), + ); + const retry = await handle( + req('/runs', { + body: { workflowId: 'open-flow', idempotencyKey: 'key-1' }, + }), + ); + + // #then one execution, and both answers describe that one run + expect(executions).toHaveLength(1); + const firstBody = (await first?.json()) as RunSummary; + const retryBody = (await retry?.json()) as RunSummary; + expect(retry?.status).toBe(200); + expect(retryBody.runId).toBe(firstBody.runId); + expect(retryBody).toEqual(firstBody); + }); + + it('starts ONE run for two same-key requests issued in parallel', async () => { + // #given the cross-isolate race a kill-and-retry harness cannot fake: two + // in-flight first calls against one database, neither having seen the other + const { handle, executions } = keyedHarness(); + + // #when + const [a, b] = await Promise.all([ + handle( + req('/runs', { + body: { workflowId: 'open-flow', idempotencyKey: 'key-1' }, + }), + ), + handle( + req('/runs', { + body: { workflowId: 'open-flow', idempotencyKey: 'key-1' }, + }), + ), + ]); + + // #then exactly one execution. That is the invariant; the loser's exact + // answer is not, and all three of its possible answers are honest: + // 200 the winner already persisted, so the loser replays it + // 503 the winner is executing, so the loser is told PENDING + // 409 the loser probed inside the CLAIM-TO-DISPATCH WINDOW — the moment + // between the winning CAS landing and the winner's start request + // reaching the object that would report it live. Nothing is running + // yet from any observer's point of view, so UNRESOLVABLE is what the + // probe can honestly say. It is a false alarm, not a lost run: the + // claim still stands, the winner still executes, and the caller's + // next retry replays the persisted summary. + // What must never appear is a second entry in `executions`. + expect(executions).toHaveLength(1); + for (const response of [a, b]) { + expect([200, 409, 503]).toContain(response?.status); + } + }); + + it('refuses the same key pointed at a different workflow', async () => { + // #given a key already spent on open-flow by an actor who may start both + const { handle, executions } = keyedHarness(); + await handle( + req('/runs', { + body: { workflowId: 'open-flow', idempotencyKey: 'key-1' }, + actor: ADMIN, + }), + ); + + // #when the SAME principal reuses it for another workflow — the same + // principal, so this is a target mismatch and not an ownership one + const response = await handle( + req('/runs', { + body: { workflowId: 'restricted-flow', idempotencyKey: 'key-1' }, + actor: ADMIN, + }), + ); + + // #then 409 naming what the key holds, and no second execution + expect(response?.status).toBe(409); + expect(await response?.json()).toMatchObject({ + reason: { + code: 'IDEMPOTENT_START_TARGET_MISMATCH', + targetId: 'open-flow', + }, + }); + expect(executions).toHaveLength(1); + }); + + it('refuses a key another principal reserved, with 403 and nothing else', async () => { + // #given + const { handle } = keyedHarness(); + await handle( + req('/runs', { + body: { workflowId: 'open-flow', idempotencyKey: 'key-1' }, + }), + ); + + // #when a different principal sends the same key + const response = await handle( + req('/runs', { + body: { workflowId: 'open-flow', idempotencyKey: 'key-1' }, + actor: ADMIN, + }), + ); + + // #then + expect(response?.status).toBe(403); + expect(await response?.json()).toMatchObject({ + reason: { code: 'IDEMPOTENT_START_OWNER_MISMATCH' }, + }); + }); + + it('answers 503 PENDING while the reserved run is still executing', async () => { + // #given a claimed reservation whose run has persisted nothing and whose + // host reports it live + const { handle, store } = keyedHarness({ live: async () => true }); + await store.reserve({ + key: 'key-1', + owner: { kind: 'human', id: OPERATOR.id }, + targetKind: 'workflow', + targetId: 'open-flow', + mintRunId: () => 'inflight_run', + }); + await store.claim('key-1', 'inflight_run'); + + // #when + const response = await handle( + req('/runs', { + body: { workflowId: 'open-flow', idempotencyKey: 'key-1' }, + }), + ); + + // #then retryable, and it names the run and when the claim was taken + expect(response?.status).toBe(503); + expect(await response?.json()).toMatchObject({ + reason: { code: 'IDEMPOTENT_START_PENDING', runId: 'inflight_run' }, + }); + }); + + it('answers 409 UNRESOLVABLE when the claim is held but nothing is running', async () => { + // #given the crash window: claim taken, nothing persisted, host gone + const { handle, store, executions } = keyedHarness({ + live: async () => false, + }); + await store.reserve({ + key: 'key-1', + owner: { kind: 'human', id: OPERATOR.id }, + targetKind: 'workflow', + targetId: 'open-flow', + mintRunId: () => 'orphan_run', + }); + await store.claim('key-1', 'orphan_run'); + + // #when + const response = await handle( + req('/runs', { + body: { workflowId: 'open-flow', idempotencyKey: 'key-1' }, + }), + ); + + // #then refused, never re-executed: whether the first step already took + // effect is unknowable, so a human decides. + expect(response?.status).toBe(409); + expect(await response?.json()).toMatchObject({ + reason: { code: 'IDEMPOTENT_START_UNRESOLVABLE', runId: 'orphan_run' }, + }); + expect(executions).toEqual([]); + }); + + it('answers 409 ALREADY_SETTLED for a key whose run completed and aged out', async () => { + // #given a settled reservation with no surviving snapshot + const { handle, store, executions } = keyedHarness(); + await store.reserve({ + key: 'key-1', + owner: { kind: 'human', id: OPERATOR.id }, + targetKind: 'workflow', + targetId: 'open-flow', + mintRunId: () => 'settled_run', + }); + await store.claim('key-1', 'settled_run'); + await store.settleRun('settled_run'); + + // #when + const response = await handle( + req('/runs', { + body: { workflowId: 'open-flow', idempotencyKey: 'key-1' }, + }), + ); + + // #then the work is done even though its outcome is gone + expect(response?.status).toBe(409); + expect(await response?.json()).toMatchObject({ + reason: { code: 'IDEMPOTENT_START_ALREADY_SETTLED' }, + }); + expect(executions).toEqual([]); + }); + + it('re-claims a reservation whose first caller died before claiming, keeping its run id', async () => { + // #given a bare reservation — nothing has executed under it + const { handle, store, executions } = keyedHarness(); + await store.reserve({ + key: 'key-1', + owner: { kind: 'human', id: OPERATOR.id }, + targetKind: 'workflow', + targetId: 'open-flow', + mintRunId: () => 'reserved_run', + }); + + // #when + const response = await handle( + req('/runs', { + body: { workflowId: 'open-flow', idempotencyKey: 'key-1' }, + }), + ); + + // #then it proceeds under the RESERVED id rather than minting a new one + expect(response?.status).toBe(200); + expect(executions).toEqual(['reserved_run']); + expect((await response?.json()) as RunSummary).toMatchObject({ + runId: 'reserved_run', + }); + }); + + it('gives the claim back when the execution fence refuses, and converges on retry after it reopens', async () => { + // #given a deployment whose fence closes between the claim and the start + let fenced = true; + const executions: string[] = []; + const { handle, store } = keyedHarness({ + start: async ({ runId }) => { + if (fenced) { + throw new ExecutionFencedError('migration-locked', 'run start'); + } + executions.push(runId); + return { runId, status: 'success' }; + }, + }); + + // #when the fenced start is refused, the operator reopens, and the client + // retries with the same key + const refused = await handle( + req('/runs', { + body: { workflowId: 'open-flow', idempotencyKey: 'key-1' }, + }), + ); + const reservedRunId = (await store.read('key-1'))?.runId; + fenced = false; + const retry = await handle( + req('/runs', { + body: { workflowId: 'open-flow', idempotencyKey: 'key-1' }, + }), + ); + + // #then the fence's own refusal reached the caller, the claim went back, + // and the retry ran the SAME run — a fence transition mid-start must not + // manufacture an unresolvable reservation, nor a second run. + expect(refused?.status).toBe(503); + expect(await refused?.json()).toMatchObject({ + reason: { code: 'EXECUTION_FENCED' }, + }); + expect(retry?.status).toBe(200); + expect(executions).toEqual([reservedRunId]); + }); + + it('keeps the claim when a start fails for any other reason', async () => { + // #given a start that failed somewhere it may already have executed + const { handle, store } = keyedHarness({ + start: async () => { + throw new Error('step exploded'); + }, + }); + + // #when + await handle( + req('/runs', { + body: { workflowId: 'open-flow', idempotencyKey: 'key-1' }, + }), + ); + + // #then the claim is held, so the next retry is refused rather than handed + // a fresh run + expect((await store.read('key-1'))?.state).toBe('started'); + }); + + it('files an approval once, and reconciles rather than re-files on the replay', async () => { + // #given a run that suspends on its first (and only) start + const reconciled: string[] = []; + const persistedSuspensions = new Set(); + const { handle, store, service } = keyedHarness({ + start: async ({ runId }) => { + persistedSuspensions.add(runId); + return suspendedSummary(runId); + }, + status: async (_workflowId, runId) => + persistedSuspensions.has(runId) ? suspendedSummary(runId) : undefined, + reconcileApprovals: async (_context, _workflowId, summary) => { + reconciled.push(summary.runId); + }, + }); + + // #when the first start files an approval and the retry replays + const first = await handle( + req('/runs', { + body: { workflowId: 'open-flow', idempotencyKey: 'key-1' }, + }), + ); + const retry = await handle( + req('/runs', { + body: { workflowId: 'open-flow', idempotencyKey: 'key-1' }, + }), + ); + const reservedRunId = (await store.read('key-1'))?.runId as string; + + // #then exactly ONE approval record exists. Re-running the start's + // unconditional filing on the replay would create a second record for the + // same gate on every retry — the exact duplication the key was bought to + // prevent — so the replay takes the deduplicating reconcile instead. + expect((await first?.json()) as { approvals?: unknown[] }).toMatchObject({ + approvals: [expect.objectContaining({ runId: reservedRunId })], + }); + expect(retry?.status).toBe(200); + expect(reconciled).toEqual([reservedRunId]); + const listed = await service.list( + { workflowId: 'open-flow', runId: reservedRunId }, + SYSTEM, + ); + expect(listed).toHaveLength(1); + }); +}); diff --git a/packages/flowsafe/src/host-kit/run-router.ts b/packages/flowsafe/src/host-kit/run-router.ts index c9911b3..3a79cde 100644 --- a/packages/flowsafe/src/host-kit/run-router.ts +++ b/packages/flowsafe/src/host-kit/run-router.ts @@ -33,12 +33,19 @@ import { RUN_START_ROLES, } from '../approval-api/index.js'; import { + beginIdempotentStart, + DoStatusError, + type ExecutionFenceWiring, InvalidRunRequestError, RunAlreadyExistsError, RunLifecycleBlockedError, RunNotSuspendedError, type RunSummary, RunTerminalConflictError, + requireStartIdempotency, + rollbackFencedStart, + type StartIdempotencyWiring, + type StartReservation, UnknownRunError, UnknownWorkflowError, } from '../do-runner/index.js'; @@ -71,6 +78,24 @@ export interface RunRouterOptions { * globally disjoint from human ids. Default: 'flowsafe-system'. */ systemPrincipalId?: string; + /** + * How this host honours `idempotencyKey` on POST /runs — the reservation + * store plus the liveness probe that resolves a claimed-but-unpersisted run. + * + * REQUIRED, and with no `undefined` in the type, for the reason + * ExecutionFenceWiring spells out: an option a host may omit is one a host + * will omit, and the failure mode of omitting THIS one is silent. A router + * that ignored an unwired key would answer an exactly-once request with + * at-least-once behaviour, and the caller would have no way to find out. The + * typed opt-out (`'none'`) is honest — it makes every keyed start refuse with + * IDEMPOTENT_START_UNSUPPORTED — and unkeyed starts are unaffected either way. + * + * The probe travels WITH the store rather than beside it because a store + * without one cannot answer the only question the reservation cannot settle + * on its own, and a host that wired the first and forgot the second would + * fall back to guessing. + */ + startIdempotency: RunRouterStartIdempotency; /** Host topology: in-process runtime, or a DO stub fetch. */ start: (input: RunStartInput) => Promise; status: ( @@ -132,6 +157,40 @@ export interface RunRouterOptions { ) => Promise; } +/** + * The router's idempotent-start wiring: either the typed opt-out, or a store + * paired with the liveness probe its replay decision depends on. + * + * `'none'` is written out as its own arm rather than allowed as a missing + * `store` field, so a host declares the absence instead of arriving at it. + */ +export type RunRouterStartIdempotency = + | 'none' + | { + store: Exclude; + /** + * Is a start for this run executing right now? Hosts wire + * `createDoRunTopology(...).startLiveness`; an in-process host wires its + * runtime's own `isRunActive`. + */ + live: (workflowId: string, runId: string) => Promise; + /** + * The deployment execution fence, so a REPLAY can re-assert a proof-only + * fence's binding to the run this key already made. + * + * Named here rather than looked up, and required rather than optional, + * because it must be the fence over the SAME database the reservation + * lives in — a second store over a different binding would silently + * re-bind nothing. `'none'` is the honest answer for a host with no fence. + * + * Named `executionFence`, like every other fence-carrying option in the + * package, so the wiring census can see it: a leaf that spelled the field + * differently would be one the census could not name, and an uncensused + * fence field is exactly the kind that quietly becomes optional. + */ + executionFence: ExecutionFenceWiring; + }; + export interface RunStartInput { workflowId: string; runId: string; @@ -142,6 +201,14 @@ export interface RunStartInput { scheduleId?: string; /** Relative deadline measured from the accepted start. */ deadlineMs?: number; + /** + * The reserved idempotency key, forwarded to the runtime on the trusted + * channel so the execution fence's proof-only state can match it. Set by this + * router from a reservation it already took — never copied straight from a + * request body, which is what keeps a tenant from naming the proof key. + * @internal + */ + idempotencyKey?: string; } export type RunRouter = (request: Request) => Promise; @@ -151,6 +218,16 @@ interface StartBody { runId?: string; inputData?: unknown; deadlineMs?: unknown; + /** + * A caller-chosen key that makes this start exactly-once for this caller. + * + * The one identifier a client MAY supply, and the reason the runId stays + * refused two fields up: a key names a REQUEST, so the worst a caller can do + * with a bad one is converge onto its own earlier run, while a runId names a + * SLOT in every store this deployment has and a caller that could choose one + * could collide with anybody's. + */ + idempotencyKey?: unknown; } // no-store is defense in depth: this is an authenticated API served from the @@ -200,6 +277,23 @@ function errorResponse(error: unknown): Response { if (error instanceof InvalidRunRequestError) { return json({ error: error.message }, 400); } + // Every refusal this package authors on the taxonomy's own base renders with + // its declared status and reason — the reservation family among them. Placed + // LAST so the named branches above keep their exact shapes, and typed against + // the base rather than against each reservation class so a refusal added + // later cannot arrive here as an anonymous 500. + if (error instanceof DoStatusError) { + const { status } = error; + if (Number.isInteger(status) && status >= 400 && status <= 599) { + return json( + { + error: error.message, + ...(error.reason === undefined ? {} : { reason: error.reason }), + }, + status, + ); + } + } return json( { error: error instanceof Error ? error.message : String(error) }, 500, @@ -225,6 +319,97 @@ async function readJson(request: Request): Promise { } } +/** + * What a keyed POST /runs resolved to: a run this request started, or one an + * earlier request already started. + * + * The distinction is not cosmetic. Filing an approval for a suspension is a + * SIDE EFFECT of starting a run, and the whole promise of an idempotency key is + * that a retry repeats none of the first request's side effects — so a replay + * must not re-file. `replayed` is what carries that fact back to the route. + */ +interface IdempotentStartResult { + summary: RunSummary; + replayed: boolean; +} + +/** + * Run a keyed start through the reservation. + * + * The reservation decides; this function only supplies the two surface reads it + * needs (a run's persisted summary, and whether its object is executing it) and + * then does what it is told. The run id comes from the reservation in BOTH + * branches — freshly minted through `context.newRunId()` on the branch that + * creates the row, read back from the winner's row on every other — so the + * server-minted-run-id rule holds unchanged: the router still mints, and the + * reservation still only ever stores what it was handed. + */ +async function startIdempotently( + options: RunRouterOptions, + context: ActorContext, + workflowId: string, + body: StartBody, + rawKey: unknown, +): Promise { + const wiring = options.startIdempotency; + // `requireStartIdempotency` turns the opt-out into the published refusal. A + // key on an unwired host is never ignored: honouring it silently would be an + // exactly-once promise this deployment cannot keep. + const store = requireStartIdempotency( + wiring === 'none' ? 'none' : wiring.store, + ); + const live = wiring === 'none' ? undefined : wiring.live; + const decision = await beginIdempotentStart( + store, + { + // No cast: `StartReservationRequest.key` takes `unknown` and `assertKey` + // is what validates it, so the one field a caller can get wrong reaches + // the check as whatever the JSON parser actually produced. + key: rawKey, + owner: { + kind: context.principal.kind, + id: context.principal.id, + }, + targetKind: 'workflow', + targetId: workflowId, + mintRunId: () => context.newRunId(), + }, + { + persisted: async (reservation: StartReservation) => + options.status(workflowId, reservation.runId), + live: async (reservation: StartReservation) => + live ? live(workflowId, reservation.runId) : false, + }, + wiring === 'none' ? undefined : wiring.executionFence, + ); + if (decision.kind === 'replay') { + return { summary: decision.persisted, replayed: true }; + } + const { runId, key } = decision.reservation; + try { + return { + summary: await options.start({ + workflowId, + runId, + inputData: body.inputData, + principal: context.principal, + idempotencyKey: key, + ...(body.deadlineMs === undefined + ? {} + : { deadlineMs: body.deadlineMs as number }), + }), + replayed: false, + }; + } catch (error) { + // Only a fence refusal gives the claim back — see rollbackFencedStart. A + // deployment that closed its fence between the claim and the start executed + // nothing, so holding the claim would turn an operator's drain into a + // permanently poisoned key; anything else may have executed, and giving the + // claim back there would hand the next retry a second run. + return rollbackFencedStart(store, key, runId, error); + } +} + export function createRunRouter(options: RunRouterOptions): RunRouter { const { workflows, resolve } = options; const systemPrincipalId = options.systemPrincipalId ?? 'flowsafe-system'; @@ -298,17 +483,58 @@ export function createRunRouter(options: RunRouterOptions): RunRouter { ); } await options.beforeStart?.(context, body.workflowId, body.inputData); - const ownedRunId = context.newRunId(); - const summary = await options.start({ - workflowId: body.workflowId, - runId: ownedRunId, - inputData: body.inputData, - principal: context.principal, - ...(body.deadlineMs === undefined - ? {} - : { deadlineMs: body.deadlineMs as number }), - }); + const startTarget = body.workflowId; + // Unkeyed starts take the path they always took: mint, start, answer. + // Keyed starts route through the reservation, which decides whether + // this request starts a run or reports one that already exists. + const { summary, replayed } = + body.idempotencyKey === undefined + ? { + summary: await options.start({ + workflowId: startTarget, + runId: context.newRunId(), + inputData: body.inputData, + principal: context.principal, + ...(body.deadlineMs === undefined + ? {} + : { deadlineMs: body.deadlineMs as number }), + }), + replayed: false, + } + : await startIdempotently( + options, + context, + startTarget, + body, + body.idempotencyKey, + ); if (summary.status !== 'suspended') return json(summary); + if (replayed) { + // A replay answers with the run's persisted state and files nothing. + // Re-running the start's approval filing here would create a SECOND + // approval record for the same gate on every retry — the exact + // duplication the key was bought to prevent — because + // queueApprovalForSuspension files unconditionally (its deduplicating + // sibling is the reconcile below, which the status route also uses). + // So the replay takes the reconcile instead: it files only a gate + // that genuinely has no open record, and is a no-op for the normal + // case where the original start already filed one. + if (options.reconcileApprovals) { + try { + await options.reconcileApprovals(context, startTarget, summary); + } catch (error) { + console.error( + JSON.stringify({ + type: 'reconcile-error', + workflowId: startTarget, + runId: summary.runId, + error: error instanceof Error ? error.message : String(error), + }), + ); + } + } + return json(summary); + } let approvals: ApprovalRecord[] = []; try { approvals = await queueApprovalForSuspension( diff --git a/packages/flowsafe/src/host-kit/stream-router.test.ts b/packages/flowsafe/src/host-kit/stream-router.test.ts index 021a4d5..34c4e91 100644 --- a/packages/flowsafe/src/host-kit/stream-router.test.ts +++ b/packages/flowsafe/src/host-kit/stream-router.test.ts @@ -10,8 +10,7 @@ // idFromName(HUB_INSTANCE_NAME); the alarm-owned path collects the publish into // pendingSends and awaits it) is proven in the final describe. It belongs to // host-approval-service.ts but is exercised here because that module's own test -// file is outside this milestone's edit scope — the wiring it proves (M-006 -// CI-M-006-004/005, DL-020) still needs coverage. +// file is outside that edit scope; the wiring it proves still needs coverage. import { describe, expect, it, vi } from 'vitest'; @@ -25,7 +24,13 @@ import { createActorResolver, InMemoryApprovalStoreFactory, } from '../approval-api/index.js'; -import { HUB_INSTANCE_NAME } from '../do-runner/index.js'; +import { + doErrorResponse, + type ExecutionFenceDatabase, + ExecutionFenceStore, + HUB_INSTANCE_NAME, +} from '../do-runner/index.js'; +import { doSummary } from './do-response.js'; import type { RunnerNamespaceLike } from './do-run-topology.js'; import { createFlowsafeWorker, @@ -37,6 +42,7 @@ import { runSlaSweepMaintenance, } from './host-approval-service.js'; import { createHubTopology, type HubNamespaceLike } from './hub-topology.js'; +import { RunRouteError } from './run-route-error.js'; import { createStreamRouter } from './stream-router.js'; import { mintStreamTicket, verifyStreamTicket } from './stream-ticket.js'; import { staticTokenVerifier } from './verifier.js'; @@ -129,7 +135,10 @@ function makeResolve(): ActorResolver { : undefined; }, storeFactory: backend, - buildService: (store) => new ApprovalService({ store }), + // In-memory store, no database to fence against: the opt-out is written down + // rather than defaulted — see ExecutionFenceWiring. + buildService: (store) => + new ApprovalService({ store, executionFence: 'none' }), }); } @@ -525,6 +534,8 @@ describe('hub fan-out wiring (host-approval-service, tested here — see file he stream: (event) => { pending.push(hubTopology.publish(event)); }, + // In-memory approval store — no database, nothing to fence. + executionFence: 'none', }); // #when — a create mutation fires the stream sink once @@ -606,3 +617,102 @@ describe('hub fan-out wiring (host-approval-service, tested here — see file he expect(order).toEqual(['publish-released', 'sweep-resolved']); }); }); + +describe('createStreamRouter run-route passthrough', () => { + function routerWith(runStatus: () => Promise) { + return createStreamRouter({ + resolve: makeResolve(), + ticketSecret: SECRET, + hub: recordingHub([]), + runner: recordingRunner([]), + runStatus, + deploymentIdentitySecret: DEPLOYMENT_IDENTITY_SECRET, + }); + } + + it('passes a 5xx refusal through with its status and reason intact', async () => { + // #given — a run-status read refused because the deployment is fenced. + // This router used to collapse every 5xx into a bare 500, so a stream + // caller could not tell a migration from a broken deployment. + const router = routerWith(() => + Promise.reject( + new RunRouteError( + 503, + "deployment execution is fenced ('migration-locked'): run resume is refused", + { code: 'EXECUTION_FENCED', state: 'migration-locked' }, + ), + ), + ); + + // #when + const response = await router( + authedPost({ channel: 'run', runId: RUN_ID, workflowId: 'wf' }), + ); + + // #then + expect(response?.status).toBe(503); + expect(await response?.json()).toEqual({ + error: + "deployment execution is fenced ('migration-locked'): run resume is refused", + reason: { code: 'EXECUTION_FENCED', state: 'migration-locked' }, + }); + }); + + it('passes an UNREADABLE fence through without the storage error behind it', async () => { + // #given — the real chain: a fence store over a faulting database, the + // refusal it raises, the DO's error mapping, and the reader that turns a DO + // response back into a RunRouteError. The passthrough forwards a 5xx + // message verbatim, so the only thing keeping the storage fault off the + // wire is that the refusal never carries it in `message`. + const secret = 'D1_ERROR: connect ECONNREFUSED 10.0.7.4:5432 db=acme-prod'; + const fence = new ExecutionFenceStore({ + prepare: () => ({ + bind: () => ({ + run: () => Promise.reject(new Error(secret)), + all: () => Promise.reject(new Error(secret)), + }), + run: () => Promise.reject(new Error(secret)), + all: () => Promise.reject(new Error(secret)), + }), + } as unknown as ExecutionFenceDatabase); + const refusal = await fence.read().catch((error: unknown) => error); + const routeError = await doSummary(doErrorResponse(refusal)).catch( + (error: unknown) => error, + ); + const router = routerWith(() => Promise.reject(routeError)); + + // #when + const response = await router( + authedPost({ channel: 'run', runId: RUN_ID, workflowId: 'wf' }), + ); + + // #then + expect(response?.status).toBe(503); + const body = await response?.text(); + expect(JSON.parse(body ?? '')).toEqual({ + error: 'execution fence state is not readable', + reason: { code: 'EXECUTION_FENCE_UNREADABLE' }, + }); + expect(body).not.toContain('ECONNREFUSED'); + expect(body).not.toContain('acme-prod'); + expect(body).not.toContain('10.0.7.4'); + }); + + it('still collapses a 5xx with NO structured reason', async () => { + // #given — the passthrough is narrow on purpose (see run-route-error.ts). + const router = routerWith(() => + Promise.reject( + new RunRouteError(502, 'upstream exploded with connection details'), + ), + ); + + // #then + const response = await router( + authedPost({ channel: 'run', runId: RUN_ID, workflowId: 'wf' }), + ); + expect(response?.status).toBe(500); + expect(await response?.json()).toEqual({ + error: 'upstream exploded with connection details', + }); + }); +}); diff --git a/packages/flowsafe/src/host-kit/stream-router.ts b/packages/flowsafe/src/host-kit/stream-router.ts index 07422a5..32d1e34 100644 --- a/packages/flowsafe/src/host-kit/stream-router.ts +++ b/packages/flowsafe/src/host-kit/stream-router.ts @@ -1,10 +1,9 @@ // SPDX-License-Identifier: Apache-2.0 -// The stream surface every streaming host mounts (DL-010, DL-015, DL-009), -// mirroring createRunRouter's shape: plain fetch routing that returns `null` for -// paths outside its ownership so the composer can compose it, and `Response` for -// everything under `/api/stream/`. Because every route is namespaced under -// `/api/stream/`, the hosts' existing `/api/*` run_worker_first entry already -// routes them — no assets-block edit (DL-015). +// The stream surface every streaming host mounts, mirroring createRunRouter's +// shape: plain fetch routing returns `null` for paths outside its ownership so +// the composer can compose it, and `Response` for everything under +// `/api/stream/`. Because every route uses that namespace, the hosts' existing +// `/api/*` run_worker_first entry routes them without an assets-block edit. // // The Worker is the SOLE ticket-verification authority. The ticket route // authenticates through the shared ActorResolver and mints a short-lived HMAC @@ -26,7 +25,7 @@ import { readBoundedBody } from '../http-body.js'; import type { RunnerNamespaceLike } from './do-run-topology.js'; import { createHubTopology, type HubNamespaceLike } from './hub-topology.js'; import { requireResourceAccess } from './resource-access.js'; -import { RunRouteError } from './run-route-error.js'; +import { RunRouteError, runRouteReason } from './run-route-error.js'; import { mintStreamTicket, verifyStreamTicket } from './stream-ticket.js'; export interface StreamRouterOptions { @@ -267,8 +266,21 @@ export function createStreamRouter(options: StreamRouterOptions): StreamRouter { if (error instanceof ActorResolutionError) { return json({ error: 'forbidden' }, 403); } - if (error instanceof RunRouteError && error.status < 500) { - return json({ error: error.message }, error.status); + if (error instanceof RunRouteError) { + // A structured reason is the DO's own published refusal code, so it + // passes through at ANY status — 5xx included, which this branch used + // to collapse into the bare 500 below. That collapse turned a 503 + // EXECUTION_FENCED into "I am broken" for every stream caller. + const reason = runRouteReason(error); + if (error.status < 500 || reason !== undefined) { + return json( + { + error: error.message, + ...(reason === undefined ? {} : { reason }), + }, + error.status, + ); + } } return json( { error: error instanceof Error ? error.message : String(error) }, diff --git a/packages/flowsafe/src/schedules/router.test.ts b/packages/flowsafe/src/schedules/router.test.ts index d0aee0b..c51f6b3 100644 --- a/packages/flowsafe/src/schedules/router.test.ts +++ b/packages/flowsafe/src/schedules/router.test.ts @@ -1,6 +1,6 @@ // SPDX-License-Identifier: Apache-2.0 -// Track D (M-006) — createScheduleRouter: the P6-lite gate order, the no-oracle -// 404s, the count + fire-rate caps, the P4 reserved-key rejection, and the audit +// createScheduleRouter: bounded ingestion gate order, no-oracle 404s, count and +// fire-rate caps, reserved-context rejection, and audit // coverage (accept + every post-auth denial; benign GET + pre-auth NOT audited). import type { @@ -10,6 +10,8 @@ import type { } from '@mastra/core/storage'; import { describe, expect, it, vi } from 'vitest'; +import { openSqlite, sqliteUnitDatabase } from '../../test-support/sqlite.js'; + import { type ActorContext, ActorResolutionError, @@ -17,6 +19,10 @@ import { type ApprovalRole, type ResourceOwner, } from '../approval-api/index.js'; +import { + type ExecutionFenceDatabase, + ExecutionFenceStore, +} from '../do-runner/index.js'; import { RunRouteError } from '../host-kit/index.js'; import { createScheduleRouter as createScheduleRouterImpl, @@ -171,6 +177,7 @@ function harness( targetPolicy?: ScheduleTargetPolicy; audit?: ScheduleRouterOptions['audit']; validateThreadTarget?: ScheduleRouterOptions['validateThreadTarget']; + executionFence?: ScheduleRouterOptions['executionFence']; } = {}, ): Harness { const store = new MemStore(); @@ -198,6 +205,9 @@ function harness( ...(overrides.validateThreadTarget !== undefined ? { validateThreadTarget: overrides.validateThreadTarget } : {}), + // 'none' is the honest wiring for MemStore — no database, nothing to fence. + // The fence cases below pass a real store. + executionFence: overrides.executionFence ?? 'none', }); const call = async (method: string, path: string, body?: unknown) => { const res = await router( @@ -230,6 +240,7 @@ describe('createScheduleRouter — gate order', () => { const router = createScheduleRouter({ resolve: resolveAs(ctx('acme', 'operator')), store: new MemStore(), + executionFence: 'none', }); expect(await router(new Request('http://host/api/other'))).toBeNull(); // sanity: our own base IS handled @@ -243,6 +254,7 @@ describe('createScheduleRouter — gate order', () => { const router = createScheduleRouter({ resolve: resolveAs(ctx('acme', 'operator')), store: new MemStore(), + executionFence: 'none', }); const res = await router( new Request('http://host/api/schedules/%', { method: 'GET' }), @@ -552,7 +564,7 @@ describe('createScheduleRouter — create', () => { expect(res.status).toBe(400); }); - it('400s + audits a reserved requestContext key (P4 barrier a)', async () => { + it('400s + audits a reserved requestContext key', async () => { const { call, events } = harness(ctx('acme', 'operator')); const res = await call('POST', '/api/schedules', { ...WORKFLOW_CREATE, @@ -601,7 +613,7 @@ describe('createScheduleRouter — create', () => { ); }); - it('400s a cron that fires faster than the fire-rate floor (DL-007)', async () => { + it('400s a cron that fires faster than the fire-rate floor', async () => { // floor 2min; a per-minute cron (60s interval) is under it const { call, events } = harness(ctx('acme', 'operator'), { minFireIntervalMs: 120_000, @@ -616,7 +628,7 @@ describe('createScheduleRouter — create', () => { ); }); - it('400s at the deployment COUNT cap (DL-007)', async () => { + it('400s at the deployment COUNT cap', async () => { const { call, events } = harness(ctx('acme', 'operator'), { maxSchedules: 1, }); @@ -655,6 +667,7 @@ describe('createScheduleRouter — resource-scoped reads', () => { const router = createScheduleRouter({ resolve: resolveAs(ctx('xyz', 'operator', async () => false)), store, + executionFence: 'none', }); const response = await router( new Request(`http://host/api/schedules/${id}`), @@ -667,6 +680,7 @@ describe('createScheduleRouter — resource-scoped reads', () => { const router = createScheduleRouter({ resolve: resolveAs(ctx('review', 'viewer', async () => true)), store, + executionFence: 'none', }); const response = await router( new Request(`http://host/api/schedules/${id}`), @@ -680,6 +694,7 @@ describe('createScheduleRouter — resource-scoped reads', () => { createScheduleRouter({ resolve: resolveAs(context), store, + executionFence: 'none', }); const ids: string[] = []; for (const actorLabel of ['acme', 'xyz']) { @@ -970,6 +985,7 @@ describe('createScheduleRouter internal errors', () => { const router = createScheduleRouter({ resolve: resolveAs(ctx('acme', 'operator')), store, + executionFence: 'none', }); try { @@ -983,3 +999,129 @@ describe('createScheduleRouter internal errors', () => { } }); }); + +describe('createScheduleRouter and the deployment execution fence', () => { + it('will not compile without explicit fence wiring', () => { + // A TYPE-level pin on the forcing function, and the representative for the + // whole required-`executionFence` sweep: the compile error is what stops a + // host wiring the runtime's fence and forgetting a router's, which is the + // partially-fenced deployment the option exists to prevent. + // + // An unused suppression directive is itself an error in this package's + // tsconfig, so `tsc` exiting 0 is what proves the negative. (The directive + // below must be the only one in this comment block — a prose line that + // BEGINS with the directive text is parsed as one.) + const build = (): unknown => + // @ts-expect-error a schedule router must state its fence wiring + createScheduleRouterImpl({ + resolve: resolveAs(ctx('acme', 'operator')), + store: new MemStore(), + targetPolicy: TARGET_POLICY, + validateThreadTarget: async () => undefined, + }); + expect(build).toBeTypeOf('function'); + }); + + async function drainingFence(): Promise { + const fence = new ExecutionFenceStore( + sqliteUnitDatabase(openSqlite()) as ExecutionFenceDatabase, + ); + await fence.seed('draining'); + return fence; + } + + function unreadableFence(): ExecutionFenceStore { + // Storage that faults on every query — NOT the "no such table" a pre-0.20 + // database answers with, which legitimately reads as open. + return new ExecutionFenceStore({ + prepare: () => ({ + bind: () => ({ + bind: () => { + throw new Error('unreachable'); + }, + run: () => Promise.reject(new Error('D1_ERROR: network')), + all: () => Promise.reject(new Error('D1_ERROR: network')), + }), + run: () => Promise.reject(new Error('D1_ERROR: network')), + all: () => Promise.reject(new Error('D1_ERROR: network')), + }), + } as unknown as ExecutionFenceDatabase); + } + + it('degrades a mutation closed with 503 when the fence cannot be read', async () => { + // #given + const { store, call } = harness(ctx('acme', 'operator'), { + executionFence: unreadableFence(), + }); + + // #then — never the generic 500: an operator must be able to tell a + // deployment being migrated from a broken one, and the write did not land. + const res = await call('POST', '/api/schedules', WORKFLOW_CREATE); + expect(res.status).toBe(503); + expect(res.body.reason).toEqual({ code: 'EXECUTION_FENCE_UNREADABLE' }); + expect(store.m.size).toBe(0); + }); + + it('refuses create, update, and resume once the deployment is draining', async () => { + // #given + const executionFence = await drainingFence(); + const { store, events, call } = harness(ctx('acme', 'operator'), { + executionFence, + }); + store.m.set('s1', { + id: 's1', + target: { type: 'workflow', workflowId: 'wf', inputData: {} }, + cron: '*/5 * * * *', + status: 'paused', + nextFireAt: 0, + createdAt: 0, + updatedAt: 0, + metadata: {}, + } as Schedule); + + // #when / #then — every operation that ARMS a future fire is refused with + // the taxonomy's retryable status and code. + for (const [method, path, body] of [ + ['POST', '/api/schedules', WORKFLOW_CREATE], + ['PATCH', '/api/schedules/s1', { cron: '*/10 * * * *' }], + ['POST', '/api/schedules/s1/resume', undefined], + ] as const) { + const res = await call(method, path, body); + expect(res.status).toBe(503); + expect(res.body.reason).toEqual({ + code: 'EXECUTION_FENCED', + state: 'draining', + }); + } + expect(store.m.size).toBe(1); + expect( + events.filter((event) => event.reason === 'execution-fenced'), + ).toHaveLength(3); + }); + + it('keeps pause, delete, and every read available while draining', async () => { + // #given — pause and delete TAKE WORK AWAY, which is the direction a drain + // is going, and a read moves nothing. + const executionFence = await drainingFence(); + const { store, call } = harness(ctx('acme', 'operator'), { + executionFence, + }); + store.m.set('s1', { + id: 's1', + target: { type: 'workflow', workflowId: 'wf', inputData: {} }, + cron: '*/5 * * * *', + status: 'active', + nextFireAt: 0, + createdAt: 0, + updatedAt: 0, + metadata: {}, + } as Schedule); + + // #then + expect((await call('GET', '/api/schedules')).status).toBe(200); + expect((await call('GET', '/api/schedules/s1')).status).toBe(200); + expect((await call('POST', '/api/schedules/s1/pause')).status).toBe(200); + expect((await call('DELETE', '/api/schedules/s1')).status).toBe(200); + expect(store.m.size).toBe(0); + }); +}); diff --git a/packages/flowsafe/src/schedules/router.ts b/packages/flowsafe/src/schedules/router.ts index 57535af..55239b3 100644 --- a/packages/flowsafe/src/schedules/router.ts +++ b/packages/flowsafe/src/schedules/router.ts @@ -1,23 +1,22 @@ // SPDX-License-Identifier: Apache-2.0 -// Track D (M-006), CI-M-006-003 — createScheduleRouter, the authenticated facade -// over the deployment schedules domain. Ids are -// SERVER-MINTED `${prefix}${uuid}` (agent_/schedule_) — a client cannot name the -// id, so it cannot collide with (or probe for) another schedule, the -// INV-1 posture applied to schedule ids; core's slugified client-id path is -// avoided (a slugify-drift + existence-oracle vector). +// createScheduleRouter, the authenticated facade over the deployment schedules +// domain. Ids are SERVER-MINTED `${prefix}${uuid}` (agent_/schedule_): a client +// cannot name the id, so it cannot collide with or probe for another schedule. +// The host-owned run-id posture applies to schedule ids; core's slugified +// client-id path is avoided because it exposes slug drift and existence probes. // -// The write path is an ingestion trust boundary (P6-lite, DL-006) — the same gate -// order createSignalRouter / createObjectiveRouter enforce: +// The write path is an ingestion trust boundary with the same gate order that +// createSignalRouter and createObjectiveRouter enforce: // -// 1. resolve (authenticate and validate actor) -> 401 / 403 -// 2. existing-schedule ownership before role or storage -> 404 -// 3. coarse role (RUN_START_ROLES) on MUTATIONS -> 403 (reads stay coarse) -// 4. size cap on the raw body, THEN JSON parse -> 413 / 400 -// 5. field validation + deployment COUNT cap + fire-RATE cap (DL-007) -// 6. P4 reserved requestContext-key rejection (DL-004) -> 400 +// 1. resolve and validate the actor -> 401 / 403 +// 2. verify existing-schedule ownership -> 404 +// 3. require RUN_START_ROLES for mutations -> 403 +// 4. cap the raw body, then parse JSON -> 413 / 400 +// 5. validate fields, count cap, and rate cap -> 400 +// 6. reject reserved request-context keys -> 400 // 7. audit (schedule.route) + persist // -// P4 STORED-CONTEXT BARRIER (a) (DL-004): a stored WorkflowSchedule.requestContext +// STORED-CONTEXT BARRIER: a stored WorkflowSchedule.requestContext // / agent ScheduleStreamOptions.requestContext replays into a future run, so it is // a stored-capability channel — the same class as the approval create-route leak. // Create/update REJECT any requestContext naming a reserved key (the whole @@ -64,6 +63,13 @@ import { type ResourceOwner, RUN_START_ROLES, } from '../approval-api/index.js'; +import { + admitsWorkAuthoring, + type ExecutionFenceWiring, + executionFencedResponse, + isExecutionFenceRefusal, + readExecutionFence, +} from '../do-runner/index.js'; import { type BoundThreadTargetValidator, RunRouteError, @@ -168,6 +174,20 @@ export interface ScheduleRouterOptions { * zero denies every non-empty body. Default 16384 (16 KiB). */ maxContentBytes?: number; + /** + * The deployment execution fence, or `'none'` for a router with no database + * behind it. REQUIRED: a router receives a store facade and a resolver, not a + * database, so it cannot build one for itself the way `init({ DB })` can — + * which leaves the host as the only place the wiring can happen, and an + * option a host may omit is one a host will omit. See ExecutionFenceWiring + * for the split-brain this closes (an unfenced surface next to a fenced + * runtime consumes work it then cannot run). + * + * The gate itself is the runtime's refusal made earlier: an operator draining + * a deployment sees a schedule create refused at the API instead of accepted + * and then never fired. + */ + executionFence: ExecutionFenceWiring; /** Route prefix. Default '/api/schedules'. */ basePath?: string; } @@ -330,7 +350,7 @@ function checkFireRate( if (!first.ok) return first.error; const second = nextFireOrReject(cron, timezone, first.value); if (!second.ok) return second.error; - // NOTE (DL-007 limitation): this samples only the NEXT two fires at request + // NOTE: this samples only the NEXT two fires at request // time. A non-uniform cron (e.g. a dense minute cluster + a long gap) can pass // when sampled during the gap yet still fire the dense cluster later; the // deployment COUNT cap and the tick's run-cap seam are the real aggregate @@ -675,10 +695,21 @@ function buildCreateRow( }; } +/** + * The operations the execution fence blocks past `open`: every one of them + * ARMS a future fire. `pause` and `delete` are deliberately absent — they take + * work away, which is what a drain wants — and so are the three reads. + */ +const FENCE_GATED_SCHEDULE_OPERATIONS = new Set([ + 'create', + 'update', + 'resume', +]); + export function createScheduleRouter( options: ScheduleRouterOptions, ): ScheduleRouter { - const { resolve, store, targetPolicy } = options; + const { executionFence, resolve, store, targetPolicy } = options; const roles = options.roles ?? RUN_START_ROLES; const maxSchedules = nonnegativeSafeInteger( options.maxSchedules ?? 100, @@ -778,6 +809,22 @@ export function createScheduleRouter( return json({ error: 'forbidden' }, 403); } + // The execution fence, after authentication so a refusal is auditable + // and tells an anonymous caller nothing about the deployment's state. + // Only the operations that ARM future fires are gated: pause and delete + // remove work, which is the direction a drain is going, and reads stay + // open in every state. + if (FENCE_GATED_SCHEDULE_OPERATIONS.has(operation)) { + const reading = await readExecutionFence(executionFence); + if (!admitsWorkAuthoring(reading)) { + await audit('rejected', 'execution-fenced'); + return executionFencedResponse( + reading.state, + `schedule ${operation}`, + ); + } + } + // LIST returns only schedules this principal may read. Reviewer/admin // policy is applied by canAccessResource; owners always retain access. if (operation === 'list') { @@ -985,6 +1032,17 @@ export function createScheduleRouter( await auditCommittedMutation(); return json({ schedule: toView(updated) }); } catch (error) { + // A fence that could not be READ is not evidence the deployment is open, + // so it degrades closed with its own retryable 503 rather than the + // generic 500 below — an operator must be able to tell a deployment + // that is being migrated from one that is broken. + if (isExecutionFenceRefusal(error)) { + await audit('rejected', 'execution-fence-unreadable'); + return json( + { error: error.message, reason: error.reason }, + error.status, + ); + } if (error instanceof RunRouteError) { await audit( 'rejected', diff --git a/packages/flowsafe/src/schedules/tick.test.ts b/packages/flowsafe/src/schedules/tick.test.ts index 78c392c..fbca6bd 100644 --- a/packages/flowsafe/src/schedules/tick.test.ts +++ b/packages/flowsafe/src/schedules/tick.test.ts @@ -1,10 +1,15 @@ // SPDX-License-Identifier: Apache-2.0 -// Track D (M-006) — createScheduleTick: the mint posture (INV-1), the run-cap -// seam (D-S4), optional agent start, fail-closed fallback, in-process single-claim -// (CAS), lost-claim classification, and the P4 stored-context barrier (b). +// createScheduleTick: mint posture, run-cap seam, optional agent start, +// fail-closed fallback, in-process single-claim CAS, lost-claim classification, +// and the stored-context barrier. import type { Schedule, ScheduleTrigger } from '@mastra/core/storage'; import { describe, expect, it, vi } from 'vitest'; +import { openSqlite, sqliteUnitDatabase } from '../../test-support/sqlite.js'; +import { + type ExecutionFenceDatabase, + ExecutionFenceStore, +} from '../do-runner/index.js'; import type { ScheduleFireClaim } from './schedules-d1.js'; import { createScheduleTargetPolicy } from './target-policy.js'; import { @@ -32,14 +37,25 @@ const TARGET_POLICY = createScheduleTargetPolicy({ ], }); +/** + * The tick under test with the shared options defaulted. + * `executionFence: 'none'` is the honest wiring for the in-memory FakeStore — + * there is no database to fence — and the fence cases at the bottom of this + * file pass a real store, so nothing here weakens their gate. + */ function createScheduleTick( - options: Omit & { + options: Omit< + ScheduleTickOptions, + 'targetPolicy' | 'status' | 'executionFence' + > & { status?: ScheduleTickOptions['status']; + executionFence?: ScheduleTickOptions['executionFence']; }, ) { return createScheduleTickImpl({ status: async () => undefined, ...options, + executionFence: options.executionFence ?? 'none', targetPolicy: TARGET_POLICY, }); } @@ -315,7 +331,7 @@ describe('createScheduleStartSource', () => { }); describe('createScheduleTick', () => { - it('fires a due workflow target through the start seam with a fresh INV-1 runId', async () => { + it('fires a due workflow target through the start seam with a fresh host-owned runId', async () => { // #given a due workflow schedule const store = new FakeStore(); store.seed( @@ -372,6 +388,7 @@ describe('createScheduleTick', () => { start, status: async () => undefined, targetPolicy, + executionFence: 'none', now: () => NOW, })(); @@ -406,7 +423,7 @@ describe('createScheduleTick', () => { ); }); - it('strips reserved requestContext keys before handing the leg context to start (P4 barrier b)', async () => { + it('strips reserved requestContext keys before handing the leg context to start', async () => { // #given a due schedule whose STORED requestContext carries a reserved key // (a compromised/tampered row) plus a benign one const store = new FakeStore(); @@ -442,7 +459,7 @@ describe('createScheduleTick', () => { expect(passedContext).toEqual({ 'my.custom': 'kept' }); }); - it('SKIPS a capped deployment (audited) but leaves the schedule healthy — the CAS already advanced it (D-S4)', async () => { + it('SKIPS a capped deployment but leaves the schedule healthy after the CAS advances it', async () => { // #given a due workflow schedule and a run cap that DENIES const store = new FakeStore(); store.seed(workflowSchedule()); @@ -1085,7 +1102,7 @@ describe('createScheduleTick', () => { }); }); -describe('the P4 reserved-key barrier helpers', () => { +describe('reserved-key barrier helpers', () => { it('isReservedScheduleContextKey covers the whole breakwater namespace + the goal key', () => { expect(isReservedScheduleContextKey('breakwater.connectorGrants')).toBe( true, @@ -1153,3 +1170,108 @@ describe('the P4 reserved-key barrier helpers', () => { }); }); }); + +describe('createScheduleTick and the deployment execution fence', () => { + function fence(): ExecutionFenceStore { + return new ExecutionFenceStore( + sqliteUnitDatabase(openSqlite()) as ExecutionFenceDatabase, + ); + } + + it.each([ + 'draining', + 'migration-locked', + ] as const)('leaves a due schedule ROW untouched on a %s pass, then fires it exactly once after reopen', async (state) => { + // #given — a due schedule on a deployment the operator has just fenced. + const store = new FakeStore(); + store.seed(workflowSchedule()); + const before = { ...(store.schedules.get('schedule_a') as Schedule) }; + const start = vi.fn(async ({ runId }: { runId: string }) => ({ runId })); + const executionFence = fence(); + await executionFence.seed(state); + const tick = createScheduleTick({ + store, + start, + executionFence, + now: () => NOW, + }); + + // #when + const fenced = await tick(); + + // #then — the pass did nothing at all. Claiming a fire it will not run + // would CONSUME it (the claim advances nextFireAt) and the fenced runtime + // would then refuse the start, so the fire would be LOST rather than + // deferred. An idle-looking RESULT is not evidence of that: the tally + // below is what the pass reported, and the row is what it did. + expect(fenced).toEqual({ + due: 0, + fired: 0, + skipped: 0, + failed: 0, + deferred: 0, + reconciled: 0, + lost: 0, + }); + expect(start).not.toHaveBeenCalled(); + expect(store.triggers).toEqual([]); + // The whole row, not just nextFireAt: status, lastFireAt, and lastRunId + // are the other fields a claim writes, and a claim that advanced any of + // them has consumed the fire whatever the tally said. + expect(store.schedules.get('schedule_a')).toEqual(before); + + // #when — the migration finishes and the operator reopens the fence. + await executionFence.transition({ expected: state, next: 'open' }); + const reopened = await tick(); + + // #then — the SAME fire runs, exactly once: neither lost nor duplicated. + expect(reopened.due).toBe(1); + expect(reopened.fired).toBe(1); + expect(start).toHaveBeenCalledTimes(1); + // #and — only NOW is the fire consumed: the row advanced past this due + // time, so a third pass would not run it again. + expect(store.schedules.get('schedule_a')?.nextFireAt).toBeGreaterThan(NOW); + }); + + it('skips the pass rather than claiming when the fence cannot be read', async () => { + // #given — a fence whose storage is down. This runs on a maintenance + // alarm, so degrading closed means doing NOTHING, not throwing: a throw + // would fail the duty, and proceeding would claim fires on a deployment + // whose state is unknown. + const store = new FakeStore(); + store.seed(workflowSchedule()); + const start = vi.fn(async ({ runId }: { runId: string }) => ({ runId })); + const unreadable = new ExecutionFenceStore({ + prepare: () => ({ + bind: () => ({ + bind: () => { + throw new Error('unreachable'); + }, + run: () => Promise.reject(new Error('D1_ERROR: network')), + all: () => Promise.reject(new Error('D1_ERROR: network')), + }), + run: () => Promise.reject(new Error('D1_ERROR: network')), + all: () => Promise.reject(new Error('D1_ERROR: network')), + }), + } as unknown as ExecutionFenceDatabase); + const log = vi.spyOn(console, 'error').mockImplementation(() => undefined); + + // #when + let result: Awaited>>; + try { + result = await createScheduleTick({ + store, + start, + executionFence: unreadable, + now: () => NOW, + })(); + } finally { + log.mockRestore(); + } + + // #then + expect(result.due).toBe(0); + expect(start).not.toHaveBeenCalled(); + expect(store.schedules.get('schedule_a')?.nextFireAt).toBe(NOW - 1000); + }); +}); diff --git a/packages/flowsafe/src/schedules/tick.ts b/packages/flowsafe/src/schedules/tick.ts index 2e4f72d..765883e 100644 --- a/packages/flowsafe/src/schedules/tick.ts +++ b/packages/flowsafe/src/schedules/tick.ts @@ -1,45 +1,46 @@ // SPDX-License-Identifier: Apache-2.0 -// Track D (M-006), CI-M-006-002 — createScheduleTick. WE OWN THE TICK (DL-012): -// a Durable Object alarm drives listDueSchedules -> CAS -// updateScheduleNextFire claim -> fire, bypassing core's pubsub worker loop -// entirely (the P1 "one chokepoint, no second execution path" rule). Core's own -// pubsub-driven schedule-worker loop is deliberately not adopted. +// createScheduleTick. WE OWN THE TICK: a Durable Object alarm drives +// listDueSchedules -> CAS updateScheduleNextFire claim -> fire, bypassing +// core's pubsub worker loop entirely under the "one chokepoint, no second +// execution path" rule. It does not adopt core's pubsub-driven schedule worker. // // TARGET KINDS: // - WORKFLOW targets: mint a fresh path-safe runId and fire // through the host's run-start seam (topology.start / RunnerRuntime.start), -// so the run inherits INV-1, the per-leg requestContext derivation, and the -// snapshot provenance. This is the fully-owned path. +// so the run inherits host-owned run ids, per-leg requestContext derivation, +// and snapshot provenance. This is the fully-owned path. // - AGENT targets: `startAgent` routes the claimed target through the host's // runtime-driven thread DO. Without that seam, the tick retains the audited // `agent-target-unsupported` skip and never adopts core's worker. // -// CAP (DL-007, P7): every unattended workflow start consults an INJECTABLE -// run-cap seam (host-agnostic; the showcase's demo caps are one implementation). -// A capped deployment yields an audited skip and the schedule stays healthy — the -// CAS claim already advanced nextFireAt, so a capped fire is CONSUMED, not -// retried hot (spike D-S4). +// CAP: every unattended workflow start consults an INJECTABLE run-cap seam. +// The showcase's demo caps are one implementation. A capped deployment yields +// an audited skip and the schedule stays healthy: the CAS claim already +// advanced nextFireAt, so the capped fire is consumed without a hot retry. // -// STORED-CONTEXT BARRIER (b) (DL-004/R-004): a schedule's stored -// WorkflowSchedule.requestContext is NEVER forwarded verbatim into a fired leg. -// The tick STRIPS every reserved key (the whole `breakwater.` namespace + core's -// goal key) before handing the remainder to the start seam. The reserved set IS -// exactly the keys #requestContextFor derives (the two scope keys + the grant -// key), so a stripped context shares NO key with the runtime-derived context — -// there is normally nothing to collide. buildScheduledLegContext keeps the -// R-004 order (stored FIRST, runtime-derived LAST) as defense-in-depth for a host -// that applies the stored context, so a reserved key that ever slipped the strip -// still LOSES to the runtime value. The DO target resolves this sanitized -// context from the exact prepared trigger snapshot and RunnerRuntime merges it -// below provider/runtime-derived values; it never trusts a forwarded body copy. +// STORED-CONTEXT BARRIER: a schedule's stored WorkflowSchedule.requestContext +// is NEVER forwarded verbatim into a fired leg. The tick STRIPS every reserved +// key: the whole `breakwater.` namespace and core's goal key, before handing +// the remainder to the start seam. The reserved set exactly matches +// #requestContextFor's keys: two scope keys and the grant key. A stripped +// context shares NO key with the runtime-derived context. There is normally +// nothing to collide. buildScheduledLegContext keeps the stored context FIRST +// and the runtime-derived context LAST. For a host applying stored context, a +// reserved key that slips the strip still LOSES to the runtime value. The DO +// target resolves this sanitized context from the exact prepared trigger +// snapshot. RunnerRuntime merges it below provider/runtime-derived values. It +// never trusts a forwarded body copy. import { ScheduleInputSchema } from '@mastra/core/schedules'; import { computeNextFireAt } from '@mastra/core/workflows'; import type { ResourceOwnershipStore } from '../approval-api/index.js'; import { + admitsWorkAuthoring, + type ExecutionFenceWiring, isPathSafeId, isReservedExecutionContextKey, RESERVED_EXECUTION_CONTEXT_KEYS, + readExecutionFence, resolveScheduleStartOwner, stripReservedExecutionContext, } from '../do-runner/index.js'; @@ -314,6 +315,18 @@ export interface ScheduleTickOptions { runCap?: ScheduleTickRunCap; /** Every fire attempt is audited through this (accepted OR skipped/failed). Absent ⇒ no audit. */ audit?: ScheduleTickAuditSink; + /** + * The deployment execution fence, read ONCE per pass and BEFORE any CAS + * claim, or `'none'` for a tick with no database behind it. + * + * REQUIRED, and this is the option the requirement exists for: a fenced pass + * must do nothing at all, because claiming a fire it will not run CONSUMES it + * — the claim advances `nextFireAt` — and the fenced runtime then refuses the + * start, so the fire is lost rather than deferred. A host that wires the + * runtime's fence but forgets this one gets exactly that, and nothing reports + * it. See ExecutionFenceWiring. + */ + executionFence: ExecutionFenceWiring; /** * Max due schedules processed per tick pass. Must be a nonnegative safe * integer; zero is an intentional no-op. Default 100. @@ -961,10 +974,10 @@ export function createScheduleTick( // claim even when a cap later consumes it without dispatch. const runId = mintPathSafeId('scheduleTick'); - // 3. CAS claim — the single-claim gate (spike D-S1). The LOSER of two + // 3. CAS claim — the single-claim gate. The LOSER of two // concurrent ticks gets false here and dispatches nothing. This serializes // the claim only; downstream dispatch remains recoverable/at-least-once. - // The cap is consulted AFTER the claim (DL-007), so `runId` is minted before + // The cap is consulted AFTER the claim, so `runId` is minted before // we know whether the fire will actually dispatch: on a capped fire the row's // `lastRunId` records this CLAIM's id though no run started. The trigger row // is the authoritative record of what happened (outcome + its own runId, @@ -1341,10 +1354,10 @@ export function createScheduleTick( return; } - // 5. Workflow target: consult the run cap (DL-007). + // 5. Workflow target: consult the run cap. const allowed = options.runCap ? await options.runCap() : true; if (!allowed) { - // Capped: the schedule stays healthy (already advanced), audited (D-S4). + // Capped: the schedule stays healthy (already advanced) and is audited. result.skipped += 1; await store.recordTrigger({ ...claimTrigger, @@ -1432,18 +1445,42 @@ export function createScheduleTick( } }; + const idlePass = (): ScheduleTickResult => ({ + due: 0, + fired: 0, + skipped: 0, + failed: 0, + deferred: 0, + reconciled: 0, + lost: 0, + }); + return async () => { - if (limit === 0) { - return { - due: 0, - fired: 0, - skipped: 0, - failed: 0, - deferred: 0, - reconciled: 0, - lost: 0, - }; + if (limit === 0) return idlePass(); + // ONE fence read per pass, before the due list and before any claim. + // Nothing at all runs on a fenced pass — not the deferred reconciliation + // either, since its agent-signal retry can dispatch. Every due fire stays + // due, so it fires when the fence reopens: neither lost nor duplicated. + // + // A fence that cannot be read skips the pass too. This is a cron/alarm + // path, so it degrades closed by DOING NOTHING and logging: throwing would + // fail the maintenance duty, and proceeding would claim fires on a + // deployment whose state is unknown. + let admitted: boolean; + try { + admitted = admitsWorkAuthoring( + await readExecutionFence(options.executionFence), + ); + } catch (error) { + console.error( + JSON.stringify({ + type: 'schedule-tick-fence-error', + reason: error instanceof Error ? error.message : String(error), + }), + ); + return idlePass(); } + if (!admitted) return idlePass(); const at = now(); const due = await store.listDueSchedules(at, limit); const result: ScheduleTickResult = { diff --git a/packages/flowsafe/src/signal-providers/host-do.test.ts b/packages/flowsafe/src/signal-providers/host-do.test.ts index e750eaf..d2b5dbc 100644 --- a/packages/flowsafe/src/signal-providers/host-do.test.ts +++ b/packages/flowsafe/src/signal-providers/host-do.test.ts @@ -6,6 +6,13 @@ import { deploymentIdentityRequest, TEST_DEPLOYMENT_IDENTITY_SECRET, } from '../../test-support/deployment-identity.js'; +import { openSqlite, sqliteUnitDatabase } from '../../test-support/sqlite.js'; +import { + type ExecutionFenceDatabase, + ExecutionFencedError, + type ExecutionFenceState, + ExecutionFenceStore, +} from '../do-runner/index.js'; import { createThreadTopology, type ThreadNamespaceLike, @@ -100,6 +107,7 @@ interface TestEnv { topology: ThreadTopology; providers: readonly SignalProviderAdapter[]; identityDatabase?: ReturnType; + executionFence?: ExecutionFenceStore; } class TestHost extends SignalProviderHost { @@ -121,6 +129,10 @@ class TestHost extends SignalProviderHost { store: env.factory.store(), topology: env.topology, providers: env.providers, + // 'none' where a case has no fence: the subscription store is backed by + // an in-memory factory, so there is no database to fence. The fence cases + // supply a real store. + executionFence: env.executionFence ?? 'none', }; } } @@ -724,3 +736,96 @@ describe('SignalProviderHost alarm + routes', () => { expect(res.status).toBe(403); }); }); + +describe('SignalProviderHost and the deployment execution fence', () => { + async function fenceAt( + state_: ExecutionFenceState, + ): Promise { + const fence = new ExecutionFenceStore( + sqliteUnitDatabase(openSqlite()) as ExecutionFenceDatabase, + ); + await fence.seed(state_); + return fence; + } + + it('refuses the REQUEST poll path with 503 once locked', async () => { + // #given + const factory = new InMemorySubscriptionStoreFactory(); + await seed(factory, 'acme', 'poller', 'acme_t1'); + const addressed: string[] = []; + const host = new TestHost(state(SIGNAL_PROVIDER_HOST_INSTANCE_NAME), { + factory, + topology: stubTopology(addressed), + providers: [pollProvider('poller', 60_000)], + executionFence: await fenceAt('migration-locked'), + }); + + // #then — a poll adapter re-reports state it has not seen accepted, so a + // refused pass costs a redelivery rather than a lost notification. + await expect(host.poll()).rejects.toBeInstanceOf(ExecutionFencedError); + expect(addressed).toEqual([]); + + // #and — over the fetch surface it is the taxonomy's 503. + const response = await host.fetch( + deploymentIdentityRequest('http://host/poll', { method: 'POST' }), + ); + expect(response.status).toBe(503); + expect(await response.json()).toMatchObject({ + reason: { code: 'EXECUTION_FENCED', state: 'migration-locked' }, + }); + }); + + it('swallows the refusal on the ALARM path and keeps its re-arm', async () => { + // #given — workerd retries a thrown alarm() up to six times, which would + // answer a deliberate operational state with a wake storm. + const factory = new InMemorySubscriptionStoreFactory(); + await seed(factory, 'acme', 'poller', 'acme_t1'); + const alarms: number[] = []; + const storage = { + setAlarm: (at: number) => { + alarms.push(at); + }, + deleteAlarm: () => undefined, + getAlarm: () => null, + }; + const addressed: string[] = []; + const host = new TestHost( + state(SIGNAL_PROVIDER_HOST_INSTANCE_NAME, storage), + { + factory, + topology: stubTopology(addressed), + providers: [pollProvider('poller', 60_000)], + executionFence: await fenceAt('migration-locked'), + }, + ); + const log = vi.spyOn(console, 'error').mockImplementation(() => undefined); + + // #when / #then + try { + await expect(host.alarm()).resolves.toBeUndefined(); + } finally { + log.mockRestore(); + } + expect(addressed).toEqual([]); + // #and — the prearm stands, so the next wake polls again once reopened. + expect(alarms).toHaveLength(1); + }); + + it('keeps polling while draining', async () => { + const factory = new InMemorySubscriptionStoreFactory(); + await seed(factory, 'acme', 'poller', 'acme_t1'); + const addressed: string[] = []; + const host = new TestHost(state(SIGNAL_PROVIDER_HOST_INSTANCE_NAME), { + factory, + topology: stubTopology(addressed), + providers: [pollProvider('poller', 60_000)], + executionFence: await fenceAt('draining'), + }); + + await expect(host.poll()).resolves.toEqual({ + providersPolled: 1, + delivered: 1, + }); + expect(addressed).toEqual(['acme_t1']); + }); +}); diff --git a/packages/flowsafe/src/signal-providers/host-do.ts b/packages/flowsafe/src/signal-providers/host-do.ts index 53f9df0..4e04606 100644 --- a/packages/flowsafe/src/signal-providers/host-do.ts +++ b/packages/flowsafe/src/signal-providers/host-do.ts @@ -4,8 +4,13 @@ import type { DurableObjectState } from '@cloudflare/workers-types'; import { + admitsDrainableExecution, DoStatusError, doErrorResponse, + ExecutionFencedError, + type ExecutionFenceWiring, + isExecutionFenceRefusal, + readExecutionFence, verifyDurableObjectDeploymentIdentity, verifyDurableObjectDeploymentRequest, } from '../do-runner/index.js'; @@ -50,6 +55,17 @@ export interface SignalProviderHostWiring { store: SubscriptionStore; topology: ThreadTopology; providers: readonly SignalProviderAdapter[]; + /** + * The deployment execution fence, or `'none'` for a host with no database + * behind it. A locked (or proof-only) deployment polls nothing: a poll + * adapter re-reports state it has not seen accepted, so a refused pass costs + * a redelivery rather than a lost notification. + * + * REQUIRED: `build()` is host code, and a host that returns wiring without a + * fence gets a poller that keeps delivering into a locked deployment. See + * ExecutionFenceWiring. + */ + executionFence: ExecutionFenceWiring; } export interface PollResult { @@ -146,14 +162,39 @@ export abstract class SignalProviderHost { await this.#withArmLock(async () => { const prearmedAt = await this.#prearmUnlocked(); await this.#verifyIdentity(); - await this.poll(); + try { + await this.poll(); + } catch (error) { + // The fence refusing (or failing to answer) is not an alarm failure: + // workerd retries a thrown alarm() up to six times, which would answer + // a deliberate operational state with a wake storm. The prearm above + // IS the re-arm, so the next wake polls again once the fence reopens — + // and #armUnlocked is deliberately skipped, because its subscription + // read would recompute a cadence from a pass that never ran. + if (!isExecutionFenceRefusal(error)) throw error; + console.error( + JSON.stringify({ + type: 'signal-provider.poll-fenced', + reason: error.message, + }), + ); + return; + } await this.#armUnlocked(prearmedAt); }); } async poll(): Promise { await this.#verifyIdentity(); - const { store, topology, providers } = this.#ensureWiring(); + const { store, topology, providers, executionFence } = this.#ensureWiring(); + // ONE fence read per PASS, before any provider is polled. Throwing rather + // than returning an empty result: the request path (POST /poll) must + // answer 503 so the caller can tell a fenced deployment from an idle one, + // and alarm() catches this exact family above. + const fence = await readExecutionFence(executionFence); + if (!admitsDrainableExecution(fence)) { + throw new ExecutionFencedError(fence.state, 'signal provider poll'); + } let providersPolled = 0; let delivered = 0; let denied = 0; diff --git a/packages/flowsafe/src/signal-providers/webhook-ingestion.integration.test.ts b/packages/flowsafe/src/signal-providers/webhook-ingestion.integration.test.ts index fd234b4..a0cc792 100644 --- a/packages/flowsafe/src/signal-providers/webhook-ingestion.integration.test.ts +++ b/packages/flowsafe/src/signal-providers/webhook-ingestion.integration.test.ts @@ -1,12 +1,11 @@ // SPDX-License-Identifier: Apache-2.0 -// Track E (M-007) E-S1 (SHOULD-FIX): one webhook through the FULL delivery chain -// with NO LLM — createWebhookRouter (verify → row lookup) → real -// createThreadTopology → real ThreadDurableObject (its principal assertion) → the -// production Track C thread signal routes → a real Agent registered with a Mastra -// whose D1 storage composes the notifications domain — and the notification LANDS -// in mastra_notifications, visible on the notifications read path. The unit suites -// each mock a seam; this wires the real seams so the ingestion boundary has one -// end-to-end proof of the webhook→inbox landing. +// One webhook through the FULL delivery chain with NO LLM: createWebhookRouter +// (verify → row lookup) → the real topology → the thread object and its +// principal assertion → production signal routes → a real Agent registered +// with a Mastra whose D1 storage composes the notifications domain. The +// notification LANDS in mastra_notifications, visible on the notifications +// read path. The unit suites each mock a seam; this wires the real seams so the +// ingestion boundary has one end-to-end proof of the webhook→inbox landing. import { Agent } from '@mastra/core/agent'; import { Mastra } from '@mastra/core/mastra'; import { MockMemory } from '@mastra/core/memory'; @@ -87,7 +86,7 @@ async function wireWebhookChain( const agent = mastra.getAgent('sig-agent'); const notifications = new D1NotificationsStorage(d1); - // A thread DO hosting the production Track C signal routes over that agent. + // A thread DO hosting the production signal routes over that agent. class TestThread extends ThreadDurableObject { readonly #threadName: string; @@ -112,7 +111,10 @@ async function wireWebhookChain( ...(contentPolicy !== undefined ? { contentPolicy } : {}), }); protected build(): InitResult { - return init({ storage }); + return init( + { storage }, + { executionFence: 'none', startIdempotency: 'none' }, + ); } protected async route( request: Request, @@ -159,6 +161,11 @@ async function wireWebhookChain( subscriptions: factory.store(), topology: createThreadTopology(namespace), secretForProvider: () => SECRET, + // Unfenced, matching the thread DO this harness drives into (its init is + // `executionFence: 'none'` above): the subject here is ingestion through to + // an owned thread, and the fence's own behavior is pinned in + // webhook-route.test.ts against a real store. + executionFence: 'none', }); return { router, threadId, notifications }; diff --git a/packages/flowsafe/src/signal-providers/webhook-route.test.ts b/packages/flowsafe/src/signal-providers/webhook-route.test.ts index 498d29d..4f2f2a8 100644 --- a/packages/flowsafe/src/signal-providers/webhook-route.test.ts +++ b/packages/flowsafe/src/signal-providers/webhook-route.test.ts @@ -1,6 +1,12 @@ // SPDX-License-Identifier: Apache-2.0 import { describe, expect, it, vi } from 'vitest'; -import { EXECUTION_PRINCIPAL_HEADER } from '../do-runner/index.js'; +import { openSqlite, sqliteUnitDatabase } from '../../test-support/sqlite.js'; +import { + EXECUTION_PRINCIPAL_HEADER, + type ExecutionFenceDatabase, + type ExecutionFenceState, + ExecutionFenceStore, +} from '../do-runner/index.js'; import { createThreadTopology as createThreadTopologyWithSecret, RunRouteError, @@ -13,12 +19,28 @@ import { type SubscriptionStoreFactory, } from './subscription-d1.js'; import { - createWebhookRouter, + createWebhookRouter as createWebhookRouterImpl, type SignalProviderAuditEvent, + type WebhookRouterOptions, } from './webhook-route.js'; const DEPLOYMENT_IDENTITY_SECRET = 'test-deployment-identity-secret-0001'; +/** + * The router under test with the fence defaulted to the honest wiring for these + * cases: the subscription store is in-memory, so there is no database to fence. + * The fence cases at the bottom of this file pass a real store. + */ +function createWebhookRouter( + options: Omit & + Partial>, +) { + return createWebhookRouterImpl({ + ...options, + executionFence: options.executionFence ?? 'none', + }); +} + /** * The parsed JSON log line of one event type — and THE only one of that type. * @@ -777,3 +799,146 @@ describe('createWebhookRouter — deployment-wide routing', () => { expect(threads.addressed).toEqual(['globex_victim']); }); }); + +describe('createWebhookRouter and the deployment execution fence', () => { + async function fenceAt( + state: ExecutionFenceState, + ): Promise { + const fence = new ExecutionFenceStore( + sqliteUnitDatabase(openSqlite()) as ExecutionFenceDatabase, + ); + await fence.seed(state); + return fence; + } + + function unreadableFence(): ExecutionFenceStore { + // Storage that faults on every query — NOT the "no such table" a pre-0.20 + // database answers with, which legitimately reads as open. + return new ExecutionFenceStore({ + prepare: () => ({ + bind: () => ({ + bind: () => { + throw new Error('unreachable'); + }, + run: () => Promise.reject(new Error('D1_ERROR: network')), + all: () => Promise.reject(new Error('D1_ERROR: network')), + }), + run: () => Promise.reject(new Error('D1_ERROR: network')), + all: () => Promise.reject(new Error('D1_ERROR: network')), + }), + } as unknown as ExecutionFenceDatabase); + } + + it('degrades closed with 503 when the fence cannot be read', async () => { + // #given + const threads = stubThreads(); + const router = createWebhookRouter({ + providers: { + test: testProvider({ + verifyWebhookSignature: () => true, + extractResourceIds: () => ['ext-1'], + }), + }, + subscriptions: new InMemorySubscriptionStoreFactory().store(), + topology: createThreadTopology(threads.namespace), + secretForProvider: () => 'secret', + executionFence: unreadableFence(), + }); + const log = vi.spyOn(console, 'error').mockImplementation(() => undefined); + + // #then — 503 so the provider's own at-least-once redelivery recovers the + // event, exactly as for a refusal. + try { + const response = await router(webhookRequest('good', { id: 'evt-1' })); + expect(response?.status).toBe(503); + expect(await response?.json()).toMatchObject({ + reason: { code: 'EXECUTION_FENCE_UNREADABLE' }, + }); + } finally { + log.mockRestore(); + } + expect(threads.addressed).toEqual([]); + }); + + it('refuses an authentic delivery with 503 once locked, so the provider redelivers', async () => { + // #given — a locked deployment and a webhook whose signature is genuine. + const threads = stubThreads(); + const extractResourceIds = vi.fn(() => ['ext-1']); + const audit = vi.fn(); + const router = createWebhookRouter({ + providers: { + test: testProvider({ + verifyWebhookSignature: () => true, + extractResourceIds, + }), + }, + subscriptions: new InMemorySubscriptionStoreFactory().store(), + topology: createThreadTopology(threads.namespace), + secretForProvider: () => 'secret', + audit, + executionFence: await fenceAt('migration-locked'), + }); + + // #when + const response = await router(webhookRequest('good', { id: 'evt-1' })); + + // #then — 503 is the status every provider retries on, so the event + // survives the migration in the PROVIDER's queue rather than half-landing + // in a database that is being copied. + expect(response?.status).toBe(503); + expect(await response?.json()).toEqual({ + error: expect.stringContaining("fenced ('migration-locked')"), + reason: { code: 'EXECUTION_FENCED', state: 'migration-locked' }, + }); + // #and — the fence runs AFTER the signature check, so it is no oracle for + // an unauthenticated caller and nothing was parsed or delivered. + expect(extractResourceIds).not.toHaveBeenCalled(); + expect(threads.addressed).toEqual([]); + expect(audit).toHaveBeenCalledWith( + expect.objectContaining({ + outcome: 'rejected', + reason: 'execution-fenced', + }), + ); + }); + + it('still rejects a FORGED signature with 401 while locked', async () => { + // #given — the verify stays first, so a forgery never learns the fence + // state and never spends the delivery path. + const router = createWebhookRouter({ + providers: { + test: testProvider({ verifyWebhookSignature: () => false }), + }, + subscriptions: new InMemorySubscriptionStoreFactory().store(), + topology: createThreadTopology(stubThreads().namespace), + secretForProvider: () => 'secret', + executionFence: await fenceAt('migration-locked'), + }); + + // #then + const response = await router(webhookRequest('bad', { id: 'evt-1' })); + expect(response?.status).toBe(401); + }); + + it('keeps delivering while draining', async () => { + // #given — draining still delivers: the thread routes degrade a wake to a + // persist there, so the inbox drains without minting. + const threads = stubThreads(); + const router = createWebhookRouter({ + providers: { + test: testProvider({ + verifyWebhookSignature: () => true, + extractResourceIds: () => [], + }), + }, + subscriptions: new InMemorySubscriptionStoreFactory().store(), + topology: createThreadTopology(threads.namespace), + secretForProvider: () => 'secret', + executionFence: await fenceAt('draining'), + }); + + // #then + const response = await router(webhookRequest('good', { id: 'evt-1' })); + expect(response?.status).toBe(200); + }); +}); diff --git a/packages/flowsafe/src/signal-providers/webhook-route.ts b/packages/flowsafe/src/signal-providers/webhook-route.ts index 131854c..32bc8ad 100644 --- a/packages/flowsafe/src/signal-providers/webhook-route.ts +++ b/packages/flowsafe/src/signal-providers/webhook-route.ts @@ -1,6 +1,6 @@ // SPDX-License-Identifier: Apache-2.0 -// Track E (M-007), CI-M-007-003 — webhook ingress that terminates on the Worker, -// plus the human-only subscribe/unsubscribe surface (DL-006/DL-017). +// Webhook ingress that terminates on the Worker, plus the human-only +// subscribe/unsubscribe surface. // // THE WEBHOOK GATE (its "auth" IS the signature, not a bearer token): // 1. path + method match (else null / 405) @@ -9,7 +9,7 @@ // 3. read the RAW bytes, size-capped (413) // 4. VERIFY the provider signature over the raw bytes — BEFORE any parse, any // subscription lookup, any delivery. A forged signature is REJECTED (401) -// and audited (E-S2). No state is touched on the reject path. +// and audited. No state is touched on the reject path. // 5. parse JSON (400 on malformed) // 6. extract the external resource key(s) from the payload // 7. map key -> deployment subscription rows — the payload NEVER names a @@ -25,8 +25,8 @@ // writing one line per attempt. Build the router ONCE per isolate (it needs no // per-request resolver) so that window persists across requests. // -// Subscribe/unsubscribe are WRITE-CLASS and stay human-only HTTP (RA-009): never -// exposed as agent tools. They mint NO capability (P8) — a subscription row is +// Subscribe/unsubscribe are WRITE-CLASS and stay human-only HTTP: never +// exposed as agent tools. They mint NO capability — a subscription row is // addressing/config, not a grant. import type { SendNotificationSignalInput } from '@mastra/core/notifications'; @@ -38,6 +38,13 @@ import { type ApprovalRole, RUN_START_ROLES, } from '../approval-api/index.js'; +import { + admitsDrainableExecution, + type ExecutionFenceWiring, + executionFencedResponse, + isExecutionFenceRefusal, + readExecutionFence, +} from '../do-runner/index.js'; import { assertNoClientMemoryIds, type BoundThreadTargetValidator, @@ -156,6 +163,19 @@ export interface WebhookRouterOptions { forgeryAuditWindowMs?: number; /** Epoch-ms clock for the forgery window, injectable for tests. Default Date.now. */ now?: () => number; + /** + * The deployment execution fence, consulted AFTER signature verification, or + * `'none'` for a router with no database behind it. A locked deployment + * answers 503 so the provider redelivers rather than treating the event as + * accepted; putting the check before the verify would turn the fence into a + * free oracle for unauthenticated callers, and would spend the forgery-audit + * budget on requests the fence refused anyway. + * + * REQUIRED: an unfenced webhook route accepts a delivery a locked deployment + * cannot forward, and a provider that saw a 2xx does not redeliver — the + * event is then lost at the migration boundary. See ExecutionFenceWiring. + */ + executionFence: ExecutionFenceWiring; } export type WebhookRouter = (request: Request) => Promise; @@ -332,6 +352,23 @@ export function createWebhookRouter( return json({ error: 'invalid signature' }, 401); } + // The execution fence, once per webhook and only for a payload already + // proven authentic. A locked (or proof-only) deployment must not ingest + // a delivery it cannot forward: 503 is the status every provider retries + // on, so the event survives the migration in the PROVIDER's queue rather + // than being half-landed in this database. Draining still delivers — + // the thread routes degrade a wake to a persist there. + const fenceReading = await readExecutionFence(options.executionFence); + if (!admitsDrainableExecution(fenceReading)) { + await auditWebhook({ + providerId, + outcome: 'rejected', + reason: 'execution-fenced', + contentBytes: rawBody.length, + }); + return executionFencedResponse(fenceReading.state, 'webhook delivery'); + } + // Parse — only now that the payload is proven authentic. let payload: unknown; try { @@ -481,6 +518,15 @@ export function createWebhookRouter( deferred > 0 ? 503 : 200, ); } catch (error) { + // A fence that could not be READ is not evidence the deployment is open. + // 503 rather than the 500 below so the provider's own at-least-once + // redelivery is what recovers the event, exactly as for a refusal. + if (isExecutionFenceRefusal(error)) { + return json( + { error: error.message, reason: error.reason }, + error.status, + ); + } console.error( JSON.stringify({ type: 'signal-provider.webhook-error', @@ -493,7 +539,7 @@ export function createWebhookRouter( }; } -// --- Subscription CRUD (human-only HTTP; RA-009) -------------------------- +// --- Subscription CRUD (human-only HTTP) ------------------------------- export interface SubscriptionRouterOptions { /** Authenticate and validate the actor; undefined means 401. */ diff --git a/packages/flowsafe/src/signals/notification-dispatch.test.ts b/packages/flowsafe/src/signals/notification-dispatch.test.ts index 53960c9..65cce0a 100644 --- a/packages/flowsafe/src/signals/notification-dispatch.test.ts +++ b/packages/flowsafe/src/signals/notification-dispatch.test.ts @@ -3,12 +3,36 @@ import { InMemoryNotificationsStorage } from '@mastra/core/notifications'; import { describe, expect, it, vi } from 'vitest'; +import { openSqlite, sqliteUnitDatabase } from '../../test-support/sqlite.js'; import type { ActorContext } from '../approval-api/index.js'; +import { + type ExecutionFenceDatabase, + type ExecutionFenceState, + ExecutionFenceStore, +} from '../do-runner/index.js'; import type { ThreadTopology } from '../host-kit/index.js'; -import { createNotificationDispatchTick } from './notification-dispatch.js'; +import { + createNotificationDispatchTick as createNotificationDispatchTickImpl, + type NotificationDispatchTickOptions, +} from './notification-dispatch.js'; const NOW = new Date('2026-07-20T12:00:00.000Z'); +/** + * The tick under test with the fence defaulted to the honest wiring for these + * cases: the notifications storage is in-memory, so there is no database to + * fence. The fence cases at the bottom of this file pass a real store. + */ +function createNotificationDispatchTick( + options: Omit & + Partial>, +) { + return createNotificationDispatchTickImpl({ + ...options, + executionFence: options.executionFence ?? 'none', + }); +} + function actorContext(groupId = 'deployment'): ActorContext { return { actor: { id: 'maintenance', role: 'admin' }, @@ -660,3 +684,85 @@ describe('createNotificationDispatchTick', () => { expect(send).not.toHaveBeenCalled(); }); }); + +describe('createNotificationDispatchTick and the deployment execution fence', () => { + async function fenceAt( + state: ExecutionFenceState, + ): Promise { + const fence = new ExecutionFenceStore( + sqliteUnitDatabase(openSqlite()) as ExecutionFenceDatabase, + ); + await fence.seed(state); + return fence; + } + + async function dueRow(): Promise { + const storage = new InMemoryNotificationsStorage(); + await storage.createNotification({ + id: 'due-1', + threadId: 'acme_thread', + resourceId: 'acme_resource', + agentId: 'agent', + source: 'test', + kind: 'ready', + summary: 'due', + deliverAt: new Date(NOW.getTime() - 1), + }); + return storage; + } + + it('skips the whole pass once locked, leaving the row due', async () => { + // #given + const storage = await dueRow(); + const send = vi.fn( + async () => new Response(JSON.stringify({ delivered: 1, failed: 0 })), + ); + const executionFence = await fenceAt('migration-locked'); + const tick = createNotificationDispatchTick({ + storage, + topology: { send } as unknown as ThreadTopology, + resolveContext: actorContext, + now: () => NOW, + executionFence, + }); + + // #when / #then — no thread DO addressed, and the row is untouched: the + // deployment taking over dispatches it. + expect(await tick()).toEqual({ due: 0, delivered: 0, failed: 0 }); + expect(send).not.toHaveBeenCalled(); + const row = await storage.getNotification({ + threadId: 'acme_thread', + id: 'due-1', + }); + expect(row?.deliveryAttempts ?? 0).toBe(0); + + // #when — reopened + await executionFence.transition({ + expected: 'migration-locked', + next: 'open', + }); + + // #then — the same row dispatches, exactly once. + expect(await tick()).toEqual({ due: 1, delivered: 1, failed: 0 }); + expect(send).toHaveBeenCalledTimes(1); + }); + + it('keeps dispatching while draining', async () => { + // #given — the thread routes degrade a wake to a persist under a drain, so + // the inbox drains without minting. + const storage = await dueRow(); + const send = vi.fn( + async () => new Response(JSON.stringify({ delivered: 1, failed: 0 })), + ); + const tick = createNotificationDispatchTick({ + storage, + topology: { send } as unknown as ThreadTopology, + resolveContext: actorContext, + now: () => NOW, + executionFence: await fenceAt('draining'), + }); + + // #then + expect(await tick()).toEqual({ due: 1, delivered: 1, failed: 0 }); + }); +}); diff --git a/packages/flowsafe/src/signals/notification-dispatch.ts b/packages/flowsafe/src/signals/notification-dispatch.ts index 60f906f..7141981 100644 --- a/packages/flowsafe/src/signals/notification-dispatch.ts +++ b/packages/flowsafe/src/signals/notification-dispatch.ts @@ -6,7 +6,12 @@ import type { } from '@mastra/core/notifications'; import type { ActorContext } from '../approval-api/index.js'; -import { isPathSafeId } from '../do-runner/index.js'; +import { + admitsDrainableExecution, + type ExecutionFenceWiring, + isPathSafeId, + readExecutionFence, +} from '../do-runner/index.js'; import type { ThreadTopology } from '../host-kit/index.js'; import { nonnegativeSafeInteger } from '../numeric-config.js'; @@ -24,6 +29,18 @@ export interface NotificationDispatchTickOptions { * intentional no-op. Values above 100 are split into route-valid chunks. */ limit?: number; + /** + * The deployment execution fence, read ONCE per pass, or `'none'` for a tick + * with no database behind it. A drain still dispatches — the thread routes + * degrade a wake to a persist there, so the inbox drains without minting — + * while migration-locked and proof-only skip the pass entirely: a due row + * stays due, so the deployment that takes over dispatches it. + * + * REQUIRED: an unfenced tick keeps delivering into thread DOs a locked + * deployment is refusing, which burns the notification's delivery attempts on + * a deployment that cannot act on it. See ExecutionFenceWiring. + */ + executionFence: ExecutionFenceWiring; } export interface NotificationDispatchTickResult { @@ -241,6 +258,27 @@ export function createNotificationDispatchTick( ); return async () => { if (limit === 0) return { due: 0, delivered: 0, failed: 0 }; + // The fence, before the due read and before any delivery. This runs on a + // maintenance alarm, so a fence that cannot be READ degrades closed by + // skipping the pass and logging: throwing would fail the duty, and + // proceeding would dispatch on a deployment whose state is unknown. + let admitted: boolean; + try { + admitted = admitsDrainableExecution( + await readExecutionFence(options.executionFence), + ); + } catch (error) { + console.error( + JSON.stringify({ + type: 'notification-dispatch-fence-error', + // `reason`, matching schedule-tick-fence-error: the two alarm lanes + // degrade closed identically, so an operator greps one field. + reason: errorMessage(error), + }), + ); + return { due: 0, delivered: 0, failed: 0 }; + } + if (!admitted) return { due: 0, delivered: 0, failed: 0 }; const now = options.now?.() ?? new Date(); const due = await options.storage.listDueNotifications({ now, diff --git a/packages/flowsafe/src/signals/notifications-d1.ts b/packages/flowsafe/src/signals/notifications-d1.ts index b901f5b..128de92 100644 --- a/packages/flowsafe/src/signals/notifications-d1.ts +++ b/packages/flowsafe/src/signals/notifications-d1.ts @@ -1,8 +1,8 @@ // SPDX-License-Identifier: Apache-2.0 -// Track C (M-004), CI-M-004-002 — the D1 NotificationsStorage domain over -// TABLE_NOTIFICATIONS ('mastra_notifications'), mirroring core's abstract -// NotificationsStorage + InMemoryNotificationsStorage reference (create / list / -// listDue / get({threadId,id}) / update + findCoalescable coalescing). Composed +// The D1 NotificationsStorage domain over TABLE_NOTIFICATIONS +// ('mastra_notifications'), mirroring core's abstract NotificationsStorage and +// InMemoryNotificationsStorage reference (create / list / listDue / +// get({threadId,id}) / update + findCoalescable coalescing). Composed // into createD1Storage's store so `agent.sendNotificationSignal` persists here // (core resolves the domain via `mastra.getStorage().getStore('notifications')`). // @@ -24,6 +24,7 @@ import { NotificationsStorage, type UpdateNotificationInput, } from '@mastra/core/notifications'; +import { DUE_NOTIFICATION_SQL } from '../do-runner/notification-predicate.js'; import { validateTablePrefix } from '../do-runner/table-prefix.js'; import { d1Changes, @@ -135,6 +136,15 @@ function isDuplicateColumn(error: unknown): boolean { return error instanceof Error && /duplicate column/i.test(error.message); } +/** + * The single-row counter table this domain allocates insertion ordinals from. + * + * Named because the deployment table census has to account for every + * `flowsafe_`-prefixed table by name, and a table whose only spelling is inside + * a template literal is one the census can only match by restating it. + */ +export const NOTIFICATION_SEQUENCE_TABLE = 'flowsafe_notification_sequence'; + export class D1NotificationsStorage extends NotificationsStorage { readonly #db: SignalDatabase; readonly #table: string; @@ -149,7 +159,7 @@ export class D1NotificationsStorage extends NotificationsStorage { const prefix = validateTablePrefix(tablePrefix) ?? ''; this.#db = db; this.#table = `${prefix}mastra_notifications`; - this.#sequenceTable = `${prefix}flowsafe_notification_sequence`; + this.#sequenceTable = `${prefix}${NOTIFICATION_SEQUENCE_TABLE}`; this.#ordinalIndex = `idx_${this.#table}_insertion_ordinal`; this.#legacyReplaceTrigger = `trg_${this.#table}_preserve_insertion_ordinal`; this.#legacyInsertTrigger = `trg_${this.#table}_allocate_insertion_ordinal`; @@ -517,10 +527,7 @@ export class D1NotificationsStorage extends NotificationsStorage { ): Promise { await this.#ensureSchema(); const now = input.now.toISOString(); - const clauses = [ - "status = 'pending'", - '((deliverAt IS NOT NULL AND deliverAt <= ?) OR (summaryAt IS NOT NULL AND summaryAt <= ?))', - ]; + const clauses = [DUE_NOTIFICATION_SQL]; const binds: unknown[] = [now, now]; if (input.agentId !== undefined) { clauses.push('agentId = ?'); diff --git a/packages/flowsafe/src/signals/signal-ingestion.integration.test.ts b/packages/flowsafe/src/signals/signal-ingestion.integration.test.ts index e415def..53bacee 100644 --- a/packages/flowsafe/src/signals/signal-ingestion.integration.test.ts +++ b/packages/flowsafe/src/signals/signal-ingestion.integration.test.ts @@ -1,8 +1,8 @@ // SPDX-License-Identifier: Apache-2.0 -// Track C (M-004) integration (SHOULD-FIX): one signal ingested through the FULL -// chain — createSignalRouter (the P6 gate) → real createThreadTopology → real -// ThreadDurableObject (its stamped-principal assertion) → the production thread -// signal routes → a runtime-driven reserve agent — with NO LLM. The unit suites +// One signal ingested through the FULL chain: createSignalRouter's ingestion +// gate → real createThreadTopology → real ThreadDurableObject (its +// stamped-principal assertion) → the production thread signal routes → a +// runtime-driven reserve agent, with NO LLM. The unit suites // each mock a seam; this one wires the real seams together so the ingestion // boundary has one end-to-end proof, including the idle-wake run cap consulted // both allowing and capping, plus a foreign path-safe thread refusal. @@ -89,7 +89,10 @@ class TestThread extends ThreadDurableObject { }); protected build(): InitResult { - return init({ storage: new InMemoryStore() }); + return init( + { storage: new InMemoryStore() }, + { executionFence: 'none', startIdempotency: 'none' }, + ); } protected async route( diff --git a/packages/flowsafe/src/signals/thread-do-routes.real-agent.test.ts b/packages/flowsafe/src/signals/thread-do-routes.real-agent.test.ts index 6862c7e..a75b509 100644 --- a/packages/flowsafe/src/signals/thread-do-routes.real-agent.test.ts +++ b/packages/flowsafe/src/signals/thread-do-routes.real-agent.test.ts @@ -126,7 +126,10 @@ function scope( return { threadId, principal: humanPrincipal({ id: 'operator', role: 'operator' }), - init: init({ storage: new InMemoryStore() }, { pubsub }), + init: init( + { storage: new InMemoryStore() }, + { pubsub, executionFence: 'none', startIdempotency: 'none' }, + ), }; } diff --git a/packages/flowsafe/src/signals/thread-do-routes.test.ts b/packages/flowsafe/src/signals/thread-do-routes.test.ts index 0162ed2..69ee2fd 100644 --- a/packages/flowsafe/src/signals/thread-do-routes.test.ts +++ b/packages/flowsafe/src/signals/thread-do-routes.test.ts @@ -1,7 +1,7 @@ // SPDX-License-Identifier: Apache-2.0 // The thread-DO signal routes (createThreadSignalRoutes): the affinity stamp // (agent.__setPubSub(scope.init.pubsub)), the delivery-decision passthrough, the -// idle run-cap consult (DL-007), and the resourceId gating — over a mock agent. +// idle run-cap consult and resourceId gating — over a mock agent. import { type Agent, @@ -17,10 +17,14 @@ import { summarizeNotifications, } from '@mastra/core/notifications'; import { describe, expect, it, vi } from 'vitest'; +import { openSqlite, sqliteUnitDatabase } from '../../test-support/sqlite.js'; import { FLOWSAFE_PERSISTENCE_FORBIDDEN } from '../agent-runner/durable-agent-runner.js'; import { RUNTIME_DRIVEN_AGENT } from '../agent-runner/index.js'; import { DoStatusError, + type ExecutionFenceDatabase, + type ExecutionFenceState, + ExecutionFenceStore, RunStateUnreadableError, type ThreadScope, } from '../do-runner/index.js'; @@ -110,7 +114,10 @@ function mockAgent( return { agent, calls, pubsub: () => stampedPubsub }; } -function scopeWith(pubsub: unknown): ThreadScope { +function scopeWith( + pubsub: unknown, + executionFence?: ExecutionFenceStore, +): ThreadScope { return { threadId: 'acme_t1', actor: { id: 'operator', role: 'operator' }, @@ -120,7 +127,7 @@ function scopeWith(pubsub: unknown): ThreadScope { role: 'operator', }, requestedBy: 'operator', - init: { pubsub }, + init: { pubsub, executionFence }, } as unknown as ThreadScope; } @@ -1304,7 +1311,7 @@ describe('createThreadSignalRoutes', () => { }); }); - it('consults the run cap for an idle WAKE and degrades to persist when over cap (DL-007)', async () => { + it('consults the run cap for an idle WAKE and degrades to persist when over cap', async () => { const { agent, calls } = mockAgent(); const routes = createThreadSignalRoutes({ resolveAgent: () => agent, @@ -3060,7 +3067,7 @@ describe('createThreadSignalRoutes', () => { await expect(responsePromise).resolves.toMatchObject({ status: 200 }); }); - it('400s a signal whose tagName is not a valid XML name (C-S5 route-level defense)', async () => { + it('400s a signal whose tagName is not a valid XML name', async () => { // #given const { agent, calls } = mockAgent(); const routes = createThreadSignalRoutes({ @@ -3143,7 +3150,7 @@ describe('createThreadSignalRoutes', () => { }); }); -describe('signalToXmlMarkup — C-S5 injection neutralization (core render pin)', () => { +describe('signalToXmlMarkup — injection neutralization', () => { // The render path the thread routes feed is core's signalToXmlMarkup. These // pin that it ENTITY-ESCAPES hostile contents and attribute values, so a // prompt-injection payload cannot break out of the element or forge a @@ -4089,3 +4096,167 @@ describe('createThreadSignalRoutes — signal content policy', () => { expect(calls).toHaveLength(1); }); }); + +describe('createThreadSignalRoutes and the deployment execution fence', () => { + async function fenceAt( + state: ExecutionFenceState, + ): Promise { + const fence = new ExecutionFenceStore( + sqliteUnitDatabase(openSqlite()) as ExecutionFenceDatabase, + ); + await fence.seed(state); + return fence; + } + + it('degrades an idle WAKE to a durable persist while draining', async () => { + // #given — a runtime-driven agent with a working start seam on a + // deployment that is draining. A drain must mint no new run, and a signal + // is the one input it cannot answer by refusing: the sender has nowhere to + // put it and the migration would lose it. + const { agent, calls } = mockAgent(); + const startIdleRun = vi.fn(async ({ runId }: { runId: string }) => ({ + runId, + signalId: 'started', + })); + const routes = createThreadSignalRoutes({ + resolveAgent: () => agent, + resolveResourceId: () => 'acme_res', + startIdleRun, + }); + + // #when — an explicit wake request. + const res = await routes( + post('/signal/message', { contents: 'hi', ifIdle: 'wake' }), + scopeWith(undefined, await fenceAt('draining')), + ); + + // #then — persisted for post-migration wake: never lost, never minted, and + // the refusal is attributed so an operator can see WHY. + expect(res?.status).toBe(200); + expect((await res?.json()) as { wakeRefused?: string }).toMatchObject({ + wakeRefused: 'execution-draining', + }); + expect(startIdleRun).not.toHaveBeenCalled(); + expect(calls[0]?.target.ifIdle).toEqual({ behavior: 'persist' }); + }); + + it('still delivers into an ACTIVE run while draining', async () => { + // #given — the run a drain is waiting for. + const { agent } = mockAgent(); + ( + agent as unknown as { getActiveThreadRunId: () => string } + ).getActiveThreadRunId = () => 'active-run'; + const routes = createThreadSignalRoutes({ + resolveAgent: () => agent, + resolveResourceId: () => 'acme_res', + }); + + // #then — delivery is what lets the drain finish, so it is admitted. + const res = await routes( + post('/signal', { contents: 'hi' }), + scopeWith(undefined, await fenceAt('draining')), + ); + expect(res?.status).toBe(200); + expect(await res?.json()).toMatchObject({ + decision: { action: 'deliver', runId: 'run-1' }, + }); + }); + + it('refuses every signal route under migration-locked, delivery and persist alike', async () => { + // #given + const { agent, calls } = mockAgent(); + ( + agent as unknown as { getActiveThreadRunId: () => string } + ).getActiveThreadRunId = () => 'active-run'; + const routes = createThreadSignalRoutes({ + resolveAgent: () => agent, + resolveResourceId: () => 'acme_res', + startIdleRun: vi.fn(), + }); + const scope = scopeWith(undefined, await fenceAt('migration-locked')); + + // #when / #then — the persist lanes are refused too: their thread-state + // write is part of what the migration is copying. + for (const path of ['/signal', '/signal/message', '/signal/queue']) { + const res = await routes(post(path, { contents: 'hi' }), scope); + expect(res?.status).toBe(503); + expect(await res?.json()).toEqual({ + error: expect.stringContaining("fenced ('migration-locked')"), + reason: { code: 'EXECUTION_FENCED', state: 'migration-locked' }, + }); + } + expect(calls).toEqual([]); + }); + + it('admits proof-only delivery to the nominated run and nothing else', async () => { + // #given — a proof state already bound to 'active-run'. + const fence = await fenceAt('migration-locked'); + await fence.transition({ + expected: 'migration-locked', + next: 'proof-only', + proofKey: 'proof-1', + }); + expect(await fence.recordProofRun('proof-1', 'active-run')).toBe(true); + const { agent } = mockAgent(); + ( + agent as unknown as { getActiveThreadRunId: () => string } + ).getActiveThreadRunId = () => 'active-run'; + const routes = createThreadSignalRoutes({ + resolveAgent: () => agent, + resolveResourceId: () => 'acme_res', + }); + + // #then — the proof run receives its signal... + const admitted = await routes( + post('/signal', { contents: 'hi' }), + scopeWith(undefined, fence), + ); + expect(admitted?.status).toBe(200); + + // #and — a thread whose active run is NOT the proof run does not. + ( + agent as unknown as { getActiveThreadRunId: () => string } + ).getActiveThreadRunId = () => 'some-other-run'; + const refused = await routes( + post('/signal', { contents: 'hi' }), + scopeWith(undefined, fence), + ); + expect(refused?.status).toBe(503); + expect(await refused?.json()).toMatchObject({ + reason: { code: 'EXECUTION_FENCED', state: 'proof-only' }, + }); + }); + + it('answers 503 rather than 502 when the fence cannot be read', async () => { + // #given — a fence whose storage is down. Degrade closed, and keep the + // status distinguishable from "the model or a route is broken". + const { agent } = mockAgent(); + const unreadable = new ExecutionFenceStore({ + prepare: () => ({ + bind: () => ({ + bind: () => { + throw new Error('unreachable'); + }, + run: () => Promise.reject(new Error('D1_ERROR: network')), + all: () => Promise.reject(new Error('D1_ERROR: network')), + }), + run: () => Promise.reject(new Error('D1_ERROR: network')), + all: () => Promise.reject(new Error('D1_ERROR: network')), + }), + } as unknown as ExecutionFenceDatabase); + const routes = createThreadSignalRoutes({ + resolveAgent: () => agent, + resolveResourceId: () => 'acme_res', + }); + + // #then + const res = await routes( + post('/signal', { contents: 'hi' }), + scopeWith(undefined, unreadable), + ); + expect(res?.status).toBe(503); + expect(await res?.json()).toMatchObject({ + reason: { code: 'EXECUTION_FENCE_UNREADABLE' }, + }); + }); +}); diff --git a/packages/flowsafe/src/signals/thread-do-routes.ts b/packages/flowsafe/src/signals/thread-do-routes.ts index 0cbfa4e..c4d0f9d 100644 --- a/packages/flowsafe/src/signals/thread-do-routes.ts +++ b/packages/flowsafe/src/signals/thread-do-routes.ts @@ -1,11 +1,10 @@ // SPDX-License-Identifier: Apache-2.0 -// Track C (M-004) — the signal routes hosted ON the thread DO (CI-M-004-001, -// DL-002). A thread's agent loop and every signal for that thread both address -// `idFromName(threadId)`, so the platform serializes them onto ONE isolate — the -// DO IS the serialization lease Mastra otherwise wants Redis for (DL-002). These -// routes run AFTER ThreadDurableObject.fetch has already verified the deployment -// identity and decoded the server-stamped principal, so `scope.threadId` and -// `scope.principal` are trusted here; the P6 ingestion gate (allowlist / size +// The signal routes hosted ON the thread DO. A thread's agent loop and every +// signal both address `idFromName(threadId)`, so the platform serializes them +// onto ONE isolate. The DO IS the serialization lease Mastra otherwise wants +// Redis for. These routes run AFTER ThreadDurableObject.fetch verifies the +// deployment identity and decodes the server-stamped principal. The scope's +// threadId and principal are trusted here; the ingestion gate (allowlist / size // cap / rate cap / audit) is the Worker-side createSignalRouter's job, the same // split createRunRouter (Worker gate) → DurableObjectRunner (execution) uses. // @@ -14,14 +13,15 @@ // module-level `defaultAgentThreadPubSub`), so a send only drains into an active // loop when BOTH run in one isolate (the DO gives this) AND both use the SAME // pubsub. The agent resolves its pubsub from `agent.getPubSub()`, so these routes -// stamp the DO's ONE identity (`scope.init.pubsub`, Track 0 / DL-001) onto the -// agent before every call — the exact reason Track A threads that same identity -// into createRun. Absent (host opted out) ⇒ core's module default, still one per +// stamp the DO's ONE identity (`scope.init.pubsub`) onto the agent before every +// call — the exact reason durable-agent execution threads that identity into +// createRun. Absent (host opted out) ⇒ core's module default, still one per // isolate, so affinity holds either way; a wired pubsub additionally makes // observe()/replay align (pubsub.ts). // // core's `agentThreadStreamRuntime` is NOT on the package exports map, so these -// routes drive the PUBLIC Agent methods only (never a deep dist import — R-001). +// routes drive only the PUBLIC Agent methods, never a deep dist import across +// the export boundary. import type { Agent, @@ -57,9 +57,14 @@ import { samePrincipal, } from '../approval-api/index.js'; import { + admitsExistingRun, DoStatusError, + type ExecutionFenceReading, + executionFencedResponse, + isExecutionFenceRefusal, isPathSafeId, type RunStatus, + readExecutionFence, type ThreadScope, } from '../do-runner/index.js'; import { internalErrorResponse } from '../internal-error-response.js'; @@ -537,6 +542,20 @@ export function createThreadSignalRoutes( if (!body) return json({ error: 'a JSON body is required' }, 400); try { + // The execution fence, read ONCE for this request and before any store + // lookup. `migration-locked` refuses every signal route outright — the + // persist lanes included, because their thread-state write is part of + // what the migration is copying. `proof-only` is decided further down, + // once the thread's active run is knowable. + // + // A read that fails throws ExecutionFenceUnreadableError, which the + // catch below answers as a 503 — degrade closed, never a silent open. + const executionFence = await readExecutionFence( + scope.init.executionFence, + ); + if (executionFence.state === 'migration-locked') { + return executionFencedResponse(executionFence.state, entryPath); + } let scheduleTarget: AgentScheduleTarget | undefined; if (path === '/signal/schedule') { if (!resolveScheduleTarget) { @@ -594,6 +613,22 @@ export function createThreadSignalRoutes( const resourceId = resolveResourceId?.(scope); const threadId = scope.threadId; + // proof-only admits work on ONE run, so the gate lives here — the first + // point where the run a signal would reach is knowable, and the only one + // every route passes through. Deciding it inside handleWake would leave + // the lanes that never reach it (the persist routes, and a default + // non-wake delivery) ungated; handleWake keeps its own check for the + // wake path it owns. + if (executionFence.state === 'proof-only') { + const activeRunId = activeThreadRunIdOf( + agent, + threadId, + resourceId ?? '', + ); + if (!admitsExistingRun(executionFence, activeRunId)) { + return executionFencedResponse(executionFence.state, entryPath); + } + } let memoryResolution: Promise | undefined; const memoryAvailable: MemoryAvailable = () => { memoryResolution ??= (async () => { @@ -668,6 +703,7 @@ export function createThreadSignalRoutes( persistenceAllowed, memoryAvailable, inspectContent, + executionFence, }, ); } @@ -701,6 +737,7 @@ export function createThreadSignalRoutes( persistenceAllowed, memoryAvailable, inspectContent, + executionFence, }, ); } @@ -714,6 +751,7 @@ export function createThreadSignalRoutes( return await handleScheduleSignal({ agent, body, + executionFence, target: scheduleTarget, threadId, resourceId, @@ -759,6 +797,7 @@ export function createThreadSignalRoutes( handleNotificationDispatch({ agent, body, + executionFence, threadId, resourceId, deploymentTag: scope.deploymentTag, @@ -794,6 +833,16 @@ export function createThreadSignalRoutes( } return json({ error: 'not found' }, 404); } catch (error) { + // The fence refusing, or failing to answer. Surfaced with its own 503 and + // reason rather than the 502 below: a caller must be able to tell "this + // deployment is deliberately not executing" (retry after the migration) + // from "the model or a route is broken". + if (isExecutionFenceRefusal(error)) { + return json( + { error: error.message, reason: error.reason }, + error.status, + ); + } if ( error instanceof DoStatusError && (error.status === 403 || error.status === 404 || error.status === 409) @@ -842,6 +891,8 @@ async function handleNotificationDispatch(options: { storage: NotificationsStorage; agentId: string; inspectContent?: InspectSignalContent; + /** The ONE fence reading this request took — see handleWake. */ + executionFence: ExecutionFenceReading; }): Promise { const ids = options.body.notificationIds; if ( @@ -986,6 +1037,7 @@ async function handleNotificationDispatch(options: { consultRunCap: options.consultRunCap, startIdleRun: options.startIdleRun, serializeWake: options.serializeWake, + executionFence: options.executionFence, blockingRun: durableBlockingRun ? () => durableBlockingRun : options.blockingRun, @@ -1170,8 +1222,17 @@ async function handleNotificationDispatch(options: { * Why a requested wake was refused. It degrades to a durable persist only when * the principal may persist and the agent has memory; otherwise the route * answers `persistence-forbidden` or `memory-unavailable`. + * + * `execution-draining` is the fence's (do-runner/execution-fence.ts): a + * draining deployment must mint no new run, and a signal is the one input a + * drain cannot answer by refusing — the sender has nowhere to put it and the + * migration would lose it. Degrading to the SAME persist branch the other two + * refusals use keeps it durable for the deployment that takes over. */ -type WakeRefusal = 'not-runtime-driven' | 'no-start-idle-run'; +type WakeRefusal = + | 'not-runtime-driven' + | 'no-start-idle-run' + | 'execution-draining'; /** Marker returned when an unbranded route cannot guarantee runtime execution. */ type RouteDegradation = 'not-runtime-driven'; @@ -1289,10 +1350,18 @@ async function handleWake(options: { * the caller's, which core's bare discard result cannot distinguish. */ activeDiscardAllowed?: boolean; + /** + * The ONE fence reading this request took (never re-read per branch). The + * route resolves it before dispatch; this function is where it is applied, + * because only here is the run a delivery would land on known — which is + * exactly what proof-only admits by. + */ + executionFence: ExecutionFenceReading; deliverActive(runId: string, memoryAvailable: boolean): WakeDelivery; persist(): WakeDelivery; }): Promise { return options.serializeWake(async () => { + const fence = options.executionFence; const durableBlockingRun = await options.blockingRun?.(); const activeRunId = activeThreadRunIdOf( options.agent, @@ -1305,6 +1374,13 @@ async function handleWake(options: { ) { return principalMismatchResponse(durableBlockingRun.runId); } + // Delivery into a run that ALREADY exists survives a drain — the run is + // what the drain is waiting for — and in proof-only it is admitted only + // for the nominated run. The check sits after the principal gate so a + // fenced deployment leaks nothing a permitted caller could not see. + if (activeRunId && !admitsExistingRun(fence, activeRunId)) { + return executionFencedResponse(fence.state, 'signal delivery'); + } if (activeRunId) { if (durableBlockingRun && durableBlockingRun.runId !== activeRunId) { return json({ @@ -1350,17 +1426,39 @@ async function handleWake(options: { }); } + // An idle thread has no run to deliver into, so from here on the only way + // to serve the signal is to MINT one. `migration-locked` and `proof-only` + // forbid that outright and have no lossless alternative to offer (the + // proof run is a specific run, not this thread's next one), so they refuse + // and the caller retries after the migration. `draining` degrades to the + // persist branch below instead — see WakeRefusal. + if (fence.state === 'migration-locked' || fence.state === 'proof-only') { + return executionFencedResponse(fence.state, 'signal wake'); + } const startIdleRun = options.startIdleRun; - const refusal: WakeRefusal | undefined = !options.runtimeDriven - ? 'not-runtime-driven' - : !startIdleRun - ? 'no-start-idle-run' - : undefined; + const refusal: WakeRefusal | undefined = + fence.state === 'draining' + ? 'execution-draining' + : !options.runtimeDriven + ? 'not-runtime-driven' + : !startIdleRun + ? 'no-start-idle-run' + : undefined; const capped = options.runtimeDriven && options.consultRunCap ? !(await options.consultRunCap()) : false; if (refusal || capped) { + // The one place a drain can lose a signal, and it is the CALLER'S choice, + // not the fence's. `execution-draining` degrades a wake into a persist so + // nothing is lost while a deployment finishes its work — but a caller who + // said persistence is forbidden has already declared it does not want its + // signal parked, and honouring the fence by parking it anyway would + // override an authorization decision with an operational one. So it is + // discarded, deliberately, and the response SAYS SO: `wakeRefused: + // 'execution-draining'` alongside `action: 'discard'` tells the caller + // exactly which condition dropped it, so a sender that would rather wait + // out the migration can retry instead of assuming delivery. if (!options.persistenceAllowed) { return persistenceForbiddenResponse({ capped, @@ -1497,6 +1595,8 @@ async function handleScheduleSignal(options: { store: ScheduleSignalDispatchStore; completed: Map; inspectContent?: InspectSignalContent; + /** The ONE fence reading this request took — see handleWake. */ + executionFence: ExecutionFenceReading; }): Promise { const scheduleId = options.body.scheduleId; const dispatchId = options.body.dispatchId; @@ -1744,6 +1844,7 @@ async function handleScheduleSignal(options: { consultRunCap: options.consultRunCap, startIdleRun: options.startIdleRun, serializeWake: options.serializeWake, + executionFence: options.executionFence, blockingRun: durableBlockingRun ? () => durableBlockingRun : options.blockingRun, @@ -1833,6 +1934,8 @@ async function handleMessage( persistenceAllowed: boolean; memoryAvailable: MemoryAvailable; inspectContent: InspectSignalContent | undefined; + /** The ONE fence reading this request took — see handleWake. */ + executionFence: ExecutionFenceReading; }, ): Promise { if (!isContents(body.contents)) { @@ -1882,6 +1985,7 @@ async function handleMessage( consultRunCap, startIdleRun, serializeWake, + executionFence: options.executionFence, blockingRun: durableBlockingRun ? () => durableBlockingRun : options.blockingRun, @@ -2007,12 +2111,14 @@ async function handleSignal( persistenceAllowed: boolean; memoryAvailable: MemoryAvailable; inspectContent: InspectSignalContent | undefined; + /** The ONE fence reading this request took — see handleWake. */ + executionFence: ExecutionFenceReading; }, ): Promise { if (!isContents(body.contents)) { return json({ error: 'contents (string) is required' }, 400); } - // Route-level tagName defense (C-S5): reject a non-XML-name tagName HERE with a + // Route-level tagName defense: reject a non-XML-name tagName HERE with a // 400, rather than letting core's signalToXmlMarkup throw at render time inside // the agent turn. Core still escapes contents/attribute values and re-validates // names; this is the ingest-time half the plan calls "route-level defense". @@ -2081,6 +2187,7 @@ async function handleSignal( consultRunCap, startIdleRun, serializeWake, + executionFence: options.executionFence, blockingRun: durableBlockingRun ? () => durableBlockingRun : options.blockingRun, diff --git a/packages/flowsafe/src/wiring-census.test.ts b/packages/flowsafe/src/wiring-census.test.ts new file mode 100644 index 0000000..5218cfb --- /dev/null +++ b/packages/flowsafe/src/wiring-census.test.ts @@ -0,0 +1,294 @@ +// SPDX-License-Identifier: Apache-2.0 +// The WIRING CENSUS: every published option type that carries a +// deployment-wide control must carry it as a REQUIRED field whose type +// includes the written opt-out. +// +// Two controls are censused here — the execution fence and the start +// reservation — in one file because they share the property that makes a +// census necessary at all. +// +// Why a census rather than a convention. Each control is only as good as its +// least-wired surface, and a missing one is invisible in exactly the way a +// wrong one is not: an unfenced router behaves identically to a fenced one +// until the day an operator closes the fence, and a router whose reservation +// store was never wired answers keyed starts exactly like a wired one right up +// to the moment a retry needed the reservation. No test written against a +// healthy deployment can catch either. What CAN catch them is the type — a +// required field is one a host cannot forget, and a union that includes the +// opt-out is one whose absence a host has to write down. This file asserts that +// property holds for every leaf at once, so a new surface cannot quietly ship +// with `executionFence?:` or `startIdempotency?:`. +// +// It is a COMPILE-TIME test. The assertions below are type aliases; `tsc` +// (pnpm --filter @proofoftech/flowsafe typecheck) is what evaluates them, and +// the negative controls are `@ts-expect-error`s — which this repo's config +// turns into errors when they are UNUSED, so a tsc that exits 0 has proved both +// directions. +// +// ADDING A SURFACE: if you add an exported `*Options` type (or any published +// wiring shape) with an `executionFence` or `startIdempotency` field, add it to +// the matching leaf list below. Nothing here can discover it for you. +// +// NOT in scope: the internal shapes that thread ONE already-taken reading down +// a request (`executionFence: ExecutionFenceReading`, thread-do-routes). Those +// carry an observation, not a wiring — the surface that took the reading is the +// leaf, and it is in the census. + +import { describe, expect, it } from 'vitest'; + +import type { AgentThreadTopologyOptions } from './agent-host/index.js'; +import type { ApprovalServiceOptions } from './approval-api/index.js'; +import type { BackgroundTaskHostOptions } from './background-tasks/index.js'; +import type { + ExecutionFenceStore, + ExecutionFenceWiring, + InitOptions, + RunnerRuntimeOptions, + StartIdempotencyStore, + StartIdempotencyWiring, + StorageInitOptions, +} from './do-runner/index.js'; +import { readExecutionFence } from './do-runner/index.js'; +import type { ObjectiveRouterOptions } from './goals/index.js'; +import type { + HostApprovalServiceOptions, + RunRouterOptions, + RunRouterStartIdempotency, +} from './host-kit/index.js'; +import type { + ScheduleRouterOptions, + ScheduleTickOptions, +} from './schedules/index.js'; +import type { + SignalProviderHostWiring, + WebhookRouterOptions, +} from './signal-providers/index.js'; +import type { NotificationDispatchTickOptions } from './signals/index.js'; + +/** Accepts only `true`; anything else is a compile error at the use site. */ +type Assert = T; + +/** + * Exact type identity, not mutual assignability. The two differ here in the way + * that matters: `ExecutionFenceStore` IS assignable to `ExecutionFenceWiring`, + * so an assignability check would pass a leaf that dropped the `'none'` arm and + * left every database-less host without a way to say so. + */ +type Equals = + (() => T extends A ? 1 : 2) extends () => T extends B ? 1 : 2 + ? true + : false; + +// --------------------------------------------------------------------------- +// The execution fence +// --------------------------------------------------------------------------- + +/** + * Does `T` carry the fence the way every leaf must? + * + * The `extends { executionFence: infer F }` does double duty. A type whose + * field is OPTIONAL does not satisfy a required one, so it fails here without a + * separate requiredness test; a type with no fence at all fails for the same + * reason. `F` is then compared exactly, which is what rejects a leaf typed as + * the store alone. + */ +type FenceWiringLeafOk = T extends { executionFence: infer F } + ? Equals + : false; + +/** + * THE FENCE CENSUS. Every exported shape that names a fence, checked in one + * place. + * + * Exported because an unused local declaration is itself a lint error here, and + * nothing consumes this one: it is checked by being written. It erases + * entirely, and the file is a test, so it reaches no published surface. + */ +export type ExecutionFenceWiringLeaves = [ + // The last leaf to be required — the service that COMMITS a decision and only + // then resumes it. + Assert>, + // The composer every fenced host builds that service through. + Assert>, + // Task bodies run below RunnerRuntime, so this is their only gate. + Assert>, + // The `{ storage }` init branch, where init cannot build a fence itself. + Assert>, + // Authoring and claiming schedule work. + Assert>, + Assert>, + // Notification dispatch — queued work a drain still finishes. + Assert>, + // Objectives: standing configuration that arms future work. + Assert>, + // Provider ingress and the provider poller behind it. + Assert>, + Assert>, + // The agent surface's keyed start, which re-binds a proof-only fence on a + // replay. The census's own instruction demanded this one and did not have it. + Assert>, + // The run router's keyed start, same job. Censused through the object arm + // because the wiring itself is a union with `'none'`, and the `'none'` arm + // carries no fields at all — see RunRouterStartIdempotencyOk below. + Assert>>, +]; + +/** + * The two published shapes that deliberately do NOT take the fence wiring, + * pinned as exceptions rather than left to be read as omissions. + * + * `InitOptions` is the `{ DB }` branch: init builds the fence from that binding + * itself, so the option is an OVERRIDE and admits no `'none'` — a host that + * hands init a database cannot end up unfenced. `RunnerRuntimeOptions` is + * construction-time state for a runtime that init always supplies, never a host + * surface. Both are typed as the store for that reason. + * + * If either is ever tightened to the required wiring, MOVE its entry up into + * the census; do not delete the check. + */ +export type DeliberateFenceWiringExceptions = [ + Assert, false>>, + Assert, false>>, +]; + +/** + * The negative controls: the three shapes the fence census exists to reject. + * Each is an `@ts-expect-error`, so an assertion that stopped biting would + * surface as an UNUSED expect-error — an error in this repo's configuration — + * rather than as a census that silently passes everything. + */ +type OptionalFence = { executionFence?: ExecutionFenceWiring }; +type StoreOnlyFence = { executionFence: ExecutionFenceStore }; +type NoFence = { store: string }; + +export type ExecutionFenceWiringRejections = [ + // @ts-expect-error an OPTIONAL fence is exactly what the census forbids + Assert>, + // @ts-expect-error a store-only fence leaves a database-less host no opt-out + Assert>, + // @ts-expect-error and a shape with no fence at all is not a wired leaf + Assert>, +]; + +// --------------------------------------------------------------------------- +// The start reservation +// --------------------------------------------------------------------------- + +/** + * Does `T` carry the reservation wiring the way every leaf must? + * + * TWO properties, not one exact type — and that is the difference from the + * fence, not a weaker rule. The fence's leaves all carry the same shape; the + * reservation's do not. The agent topology takes the bare + * `StartIdempotencyWiring`, while the run router's arm BUNDLES the store with + * the liveness probe its replay decision cannot be made without, and demanding + * one exact type would force one of those two to be wrong. + * + * What must hold for both: + * + * REQUIRED. `T extends { startIdempotency: infer F }` fails for an optional + * field — a required property must be declared — and for a missing one, so + * requiredness needs no separate test. + * + * THE OPT-OUT IS IN THE TYPE. `'none' extends F` is what makes a host with no + * store say so; without it, "no reservations here" would have no spelling and + * the field would have to go back to being optional. + * + * NO `undefined`. Checked separately, because an explicitly `| undefined` + * field is required at the type level yet reachable by writing nothing + * meaningful — the same hole an optional field opens, spelled differently. + */ +type StartIdempotencyLeafOk = T extends { startIdempotency: infer F } + ? 'none' extends F + ? Equals, never> + : false + : false; + +/** + * THE RESERVATION CENSUS. Exported for the same reason the fence's list is. + * + * Both keyed-start surfaces are here and nothing else is, because these are the + * only two places a caller's idempotency key enters the system. Everything else + * that touches the table — the runtime that settles, the purge that reaps — + * receives a store it was constructed with rather than being wired for one. + */ +export type StartIdempotencyWiringLeaves = [ + // The workflow surface: POST /runs with an idempotencyKey. + Assert>, + // The agent surface: a trusted seam starting an agent run under a key. + Assert>, + // The `{ storage }` init branch, where init cannot build a store itself. + Assert>, +]; + +/** + * Whatever shape a leaf's reservation wiring takes, the STORE inside it must be + * the real one and the opt-out must be the written `'none'`. Checked separately + * because `StartIdempotencyLeafOk` deliberately does not compare F exactly. + */ +export type RunRouterStartIdempotencyOk = [ + Assert< + Equals< + Exclude['store'], + StartIdempotencyStore + > + >, + Assert, 'none'>>, +]; + +/** + * The two published shapes that deliberately do NOT take the reservation + * wiring. Their rationale is NOT the fence's, and copying the fence's comment + * here would have been false. + * + * `InitOptions` is the `{ DB }` branch, where the option is IGNORED rather than + * overridden: init always builds the store from that binding, because the + * reservation table has to live in the database the runs live in. There is no + * third answer for a host to choose, so there is nothing for it to write. (The + * fence's `{ DB }` option, by contrast, IS honoured as an override — a host may + * share one fence store across its Durable Objects.) + * + * `RunnerRuntimeOptions` is construction-time state for a runtime init always + * supplies, never a host surface — the one point the two exceptions share. + * + * The `{ storage }` branch is deliberately NOT an exception any more: it is in + * the census above, because that is the one branch where a host CAN wire a + * store into its router and still leave the runtime unable to settle what the + * router reserved. + */ +export type DeliberateStartIdempotencyExceptions = [ + Assert, false>>, + Assert, false>>, +]; + +/** The three shapes the reservation census exists to reject. */ +type OptionalStartIdempotency = { startIdempotency?: StartIdempotencyWiring }; +type StoreOnlyStartIdempotency = { startIdempotency: StartIdempotencyStore }; +type UndefinedableStartIdempotency = { + startIdempotency: StartIdempotencyWiring | undefined; +}; + +export type StartIdempotencyWiringRejections = [ + // @ts-expect-error an OPTIONAL reservation wiring is what the census forbids + Assert>, + // @ts-expect-error a store-only field leaves a store-less host no opt-out + Assert>, + // @ts-expect-error and `| undefined` is an optional field wearing a disguise + Assert>, +]; + +describe('wiring census', () => { + it('resolves a written opt-out to the same reading an absent fence gives', async () => { + // #given — the two ways a surface ends up unfenced: a leaf that WROTE the + // opt-out the census forces it to write, and an internal seam that simply + // holds nothing. + // #when — both are resolved through the one resolver every gate uses. + const written = await readExecutionFence('none'); + const absent = await readExecutionFence(undefined); + + // #then — identical readings. This is the premise the whole census rests + // on: requiring the field costs a database-less host nothing but the words. + expect(written).toEqual(absent); + expect(written).toEqual({ state: 'open' }); + }); +}); diff --git a/packages/flowsafe/test-support/harness-probe.ts b/packages/flowsafe/test-support/harness-probe.ts index 766f733..7610c16 100644 --- a/packages/flowsafe/test-support/harness-probe.ts +++ b/packages/flowsafe/test-support/harness-probe.ts @@ -581,7 +581,7 @@ const handler = { return new Response('not found', { status: 404 }); } if (path === '/seed') { - await seedDeploymentIdentity(env.DB, 'spike'); + await seedDeploymentIdentity(env.DB, 'spike', 'open'); return Response.json({ ok: true }); } const result = diff --git a/packages/showcase/CHANGELOG.md b/packages/showcase/CHANGELOG.md index a8bc334..c7ed2bc 100644 --- a/packages/showcase/CHANGELOG.md +++ b/packages/showcase/CHANGELOG.md @@ -1,5 +1,12 @@ # showcase +## 0.0.22 + +### Patch Changes + +- Updated dependencies [1212ba5] + - @proofoftech/flowsafe@0.20.0 + ## 0.0.21 ### Patch Changes diff --git a/packages/showcase/package.json b/packages/showcase/package.json index 94999f3..bccbd95 100644 --- a/packages/showcase/package.json +++ b/packages/showcase/package.json @@ -1,6 +1,6 @@ { "name": "showcase", - "version": "0.0.21", + "version": "0.0.22", "private": true, "description": "Anchorage showcase — six runnable workflows and seven guardrail scenarios behind one React frontend", "type": "module", diff --git a/packages/showcase/worker/runtime.ts b/packages/showcase/worker/runtime.ts index 55d2fdd..9457469 100644 --- a/packages/showcase/worker/runtime.ts +++ b/packages/showcase/worker/runtime.ts @@ -87,6 +87,8 @@ export function buildShowcaseRuntime(deps: ShowcaseDeps): RunnerRuntime { // host-built (see ShowcaseDeps.grantProvider) so the store binding matches // the host's topology. requestContextForRun: deps.grantProvider, + executionFence: deps.executionFence, + startIdempotency: deps.startIdempotency, }); for (const workflowModule of SHOWCASE_MODULES) { diff --git a/packages/showcase/worker/worker.e2e.test.ts b/packages/showcase/worker/worker.e2e.test.ts index b59278e..bc8db63 100644 --- a/packages/showcase/worker/worker.e2e.test.ts +++ b/packages/showcase/worker/worker.e2e.test.ts @@ -34,6 +34,8 @@ function buildHarness() { const grants = approvalGrantProvider(store); const runtime = buildShowcaseRuntime({ initInput: { storage: new InMemoryStore() }, + executionFence: 'none', + startIdempotency: 'none', grantProvider: async (workflowId, runId, leg) => ({ ...(await grants(workflowId, runId, leg)), [AGENT_AUDIT_CONTEXT_KEY]: { @@ -48,6 +50,9 @@ function buildHarness() { }); const service = new ApprovalService({ store, + // In-memory store, no database to fence against: the opt-out is written down + // rather than defaulted — see ExecutionFenceWiring. + executionFence: 'none', resumeRun: resumeViaRuntime(runtime), }); return { runtime, service, audit }; diff --git a/packages/showcase/worker/worker.ts b/packages/showcase/worker/worker.ts index 615d9c7..0478eab 100644 --- a/packages/showcase/worker/worker.ts +++ b/packages/showcase/worker/worker.ts @@ -69,9 +69,11 @@ import { import { type DurableObjectRunLifecycleHooks, DurableObjectRunner, + executionFenceFor, HubDurableObject, type RequestContextProvider, type RunnerRuntime, + startIdempotencyFor, } from '@proofoftech/flowsafe/do-runner'; import { approvalStoreFactoryFor, @@ -143,9 +145,9 @@ interface Env { * Secret (`wrangler secret put STREAM_TICKET_SECRET`): the dedicated HS256 * key that signs short-lived WebSocket stream tickets. Present WITH the HUB * binding => the composer mounts live streaming (queue + run channels); - * absent => streaming stays unmounted and the SPA runs on polling only - * (graceful degradation, DL-019). A dedicated key so a stream ticket and a - * session JWT can never be confused under one signing secret. + * absent => streaming stays unmounted and the SPA runs on polling only. A + * dedicated key ensures a stream ticket and a session JWT can never be + * confused under one signing secret. */ STREAM_TICKET_SECRET?: string; /** Default SLA seconds for new approvals (var; default 14400 = 4h). */ @@ -225,6 +227,11 @@ function grantProviderFor(env: Env): RequestContextProvider { function defineWorkflows(env: Env): RunnerRuntime { return buildShowcaseRuntime({ initInput: env, + executionFence: executionFenceFor(env.DB), + // The same binding the composed Worker's run router reserves into, taken + // through the package's per-binding memo so the two are one store: the + // router reserves and claims, and THIS runtime is what settles. + startIdempotency: startIdempotencyFor(env.DB), // Grants are per run. The deployment tag is infrastructure-provided and // reaches Breakwater only as trusted audit correlation; connector rate and // idempotency keys are deployment-wide by physical isolation. diff --git a/packages/showcase/worker/workflows.e2e.test.ts b/packages/showcase/worker/workflows.e2e.test.ts index 43bd5be..1f82f6c 100644 --- a/packages/showcase/worker/workflows.e2e.test.ts +++ b/packages/showcase/worker/workflows.e2e.test.ts @@ -53,6 +53,8 @@ function buildHarness() { const grants = approvalGrantProvider(store); const runtime = buildShowcaseRuntime({ initInput: { storage: new InMemoryStore() }, + executionFence: 'none', + startIdempotency: 'none', grantProvider: async (workflowId, runId, leg) => ({ ...(await grants(workflowId, runId, leg)), [AGENT_AUDIT_CONTEXT_KEY]: { @@ -70,6 +72,9 @@ function buildHarness() { // product-launch flow); single-gate runs simply resume to success. const service: ApprovalService = new ApprovalService({ store, + // In-memory store, no database to fence against: the opt-out is written down + // rather than defaulted — see ExecutionFenceWiring. + executionFence: 'none', resumeRun: resumeRunWithRequeue( resumeViaRuntime(runtime), () => service, @@ -559,6 +564,7 @@ describe('showcase run routes', () => { buildService: (store) => new ApprovalService({ store, + executionFence: 'none', resumeRun: resumeRunWithRequeue( resumeViaRuntime(harness.runtime), () => harness.service, @@ -567,6 +573,9 @@ describe('showcase run routes', () => { }), }), systemPrincipalId: SYSTEM, + // In-memory harness, no database to reserve against: the opt-out is + // written down rather than defaulted — see RunRouterStartIdempotency. + startIdempotency: 'none', start: async ({ workflowId, runId, inputData, principal }) => { const resources = harness.storeFactory.resources(); const resourceOwner = principalOwner(principal); diff --git a/packages/showcase/worker/workflows/shared.ts b/packages/showcase/worker/workflows/shared.ts index e04c77f..ddb9128 100644 --- a/packages/showcase/worker/workflows/shared.ts +++ b/packages/showcase/worker/workflows/shared.ts @@ -19,8 +19,10 @@ import type { R2ArtifactStore, } from '@proofoftech/flowsafe/artifacts'; import type { + ExecutionFenceWiring, InitSource, RequestContextProvider, + StartIdempotencyWiring, } from '@proofoftech/flowsafe/do-runner'; import { z } from 'zod'; @@ -84,6 +86,22 @@ export interface EgressBinding { export interface ShowcaseDeps { /** `init` input: a Cloudflare env (D1 from `DB`) or `{ storage }`. */ initInput: InitSource; + /** + * The deployment execution fence for `initInput`. Required because the shape + * of `initInput` is only known at runtime, so nothing here can tell whether + * init would build one: a D1-backed host passes a store over the SAME + * binding, and an in-memory test host passes `'none'`. + */ + executionFence: ExecutionFenceWiring; + /** + * The deployment's start reservations for `initInput`. Required for the same + * reason the fence is: the shape of `initInput` is only known at runtime, so + * nothing here can tell whether init would build one. A D1-backed host passes + * a store over the SAME binding its run router reserves into — the runtime is + * what marks a reservation spent when its run ends — and an in-memory test + * host passes `'none'`. + */ + startIdempotency: StartIdempotencyWiring; /** * The grant-minting seam, built by the host from the deployment-wide * approval store. The provider is consulted on every start and resume leg. diff --git a/scripts/architecture-positive-controls.test.mjs b/scripts/architecture-positive-controls.test.mjs index b14f101..0054eb4 100644 --- a/scripts/architecture-positive-controls.test.mjs +++ b/scripts/architecture-positive-controls.test.mjs @@ -51,11 +51,29 @@ test('every architecture rule has an executable positive control', () => { for (const [ruleName, fixture] of Object.entries(controls)) { test(`${ruleName} rejects its positive control`, () => { - const result = spawnSync( - process.execPath, - [cli, '--config', configPath, '--output-type', 'json', fixture], - { cwd: fileURLToPath(new URL('..', import.meta.url)), encoding: 'utf8' }, - ); + const args = [ + cli, + '--config', + configPath, + '--output-type', + 'json', + fixture, + ]; + const result = spawnSync(process.execPath, args, { + cwd: fileURLToPath(new URL('..', import.meta.url)), + encoding: 'utf8', + }); + const command = [process.execPath, ...args].join(' '); + if (result.error) { + throw new Error( + `failed to spawn ${JSON.stringify(command)}: ${result.error.message}`, + ); + } + if (result.stdout.trim() === '') { + throw new Error( + `${JSON.stringify(command)} produced no JSON output (status=${String(result.status)}, signal=${String(result.signal)}); stderr=${JSON.stringify(result.stderr.slice(0, 300))}`, + ); + } assert.equal(result.signal, null, result.stderr); assert.equal(result.status, 0, result.stderr);