diff --git a/.changeset/config.json b/.changeset/config.json index 1c17a37edd..4fd58b0a5d 100644 --- a/.changeset/config.json +++ b/.changeset/config.json @@ -32,6 +32,8 @@ "workflow-sdk-compiler-playground", "@workflow/docs-typecheck", "@workflow/example-*", - "@workflow/vitest-workbench" + "@workflow/vitest-workbench", + "@workflow/world-sim", + "@workflow/sim-world-workbench" ] } diff --git a/.changeset/per-kind-correlation-ids.md b/.changeset/per-kind-correlation-ids.md deleted file mode 100644 index 2e13a31a73..0000000000 --- a/.changeset/per-kind-correlation-ids.md +++ /dev/null @@ -1,6 +0,0 @@ ---- -'@workflow/core': patch -'workflow': patch ---- - -Add env option to split correlation ID derivation into per-entity-type sequential ULIDs, instead of sharing one derivation source diff --git a/.changeset/slot-event-ids.md b/.changeset/slot-event-ids.md new file mode 100644 index 0000000000..70a6bbb897 --- /dev/null +++ b/.changeset/slot-event-ids.md @@ -0,0 +1,9 @@ +--- +'@workflow/world-postgres': patch +'@workflow/world-vercel': patch +'@workflow/world-local': patch +'@workflow/core': patch +'@workflow/world': patch +--- + +**Breaking**: SpecVersion 6: Event IDs are now a dense per-run slot number, allocated by the world at publish time so a rejected write leaves no gap in the event log. A replay tells the world how many events it had read and gets back the ones it did not see, so an event that arrives from outside the replay and lands ahead of an event the replay wrote no longer fails the run with `CORRUPTED_EVENT_LOG`: it is held for whichever part of the workflow awaits it. A gap in the numbering fails the run instead of being replayed over. diff --git a/.github/workflows/world-sim.yml b/.github/workflows/world-sim.yml new file mode 100644 index 0000000000..df62b815f7 --- /dev/null +++ b/.github/workflows/world-sim.yml @@ -0,0 +1,155 @@ +name: World Sim + +# Plays the deterministic scenario book (`workbench/sim-world`) against the +# runtime in this commit, once per log world, and publishes the two summaries. +# +# This lane never blocks a merge, by design. Six scenarios in the book fail on +# purpose: each one is a reproduction of a corruption the runtime can still +# produce, stating the outcome its own durable log implies, and staying red +# until the runtime gets there. A gate that goes red on every PR is a gate +# everyone learns to ignore, so the job publishes numbers instead of verdicts β€” +# and the number to watch is in the comment, not the check mark. +# +# mint-ordered (production): 35 passed, 6 failed, 6 violations +# append-only: 41 passed, 0 failed, 0 violations +# +# A seventh red is a regression. Five means something got fixed and a scenario +# is ready to retire. The append-only column is the measurement the pair exists +# for: it says which of the six would close if event positions were assigned at +# commit instead of at the handler's mint. + +on: + push: + branches: + - main + tags: + - "!*" + pull_request: + types: + - opened + - reopened + - synchronize + workflow_dispatch: + +concurrency: + # Unique group for this workflow and branch + group: ${{ github.workflow }}-${{ github.ref }} + cancel-in-progress: ${{ github.ref != 'refs/heads/main' }} + +permissions: + contents: read + pull-requests: write + +jobs: + world-sim: + name: World Sim + runs-on: ubuntu-latest + # Non-blocking at the job level, not just on the sim steps: a failed + # checkout, install, or PR comment should not turn this lane into a red X + # on a PR it has nothing to say about. The step summary and the artifact + # still carry whatever did happen. + continue-on-error: true + # The book itself is ~11s per world on a laptop. Everything else here is + # install and build. + timeout-minutes: 15 + env: + TURBO_TOKEN: ${{ secrets.TURBO_TOKEN }} + TURBO_TEAM: ${{ vars.TURBO_TEAM }} + + steps: + - name: Checkout Repo + uses: actions/checkout@v4 + with: + ref: ${{ github.event.pull_request.head.sha || github.sha }} + + # The default build (every package, no workbench) rather than a filter + # narrowed to the sim's own dependencies: those two graphs reach most of + # the repo anyway β€” through `@workflow/builders`, which is what compiles + # the scenario workflows β€” and matching what the other lanes build is + # what keeps this one on their turbo cache instead of warming its own. + # The workbench itself has no build; `run.ts` is executed by Node's type + # stripping. + - name: Setup environment + uses: ./.github/actions/setup-workflow-dev + + # Both worlds run with their real exit codes: `continue-on-error` records + # the non-zero without ending the job, so the step's own status still + # says whether the book was clean. Paths are absolute because pnpm runs + # the script from the package directory, not the workspace root. + - name: Play the book (mint-ordered log) + id: mint + continue-on-error: true + run: | + pnpm --filter @workflow/sim-world-workbench sim \ + --no-color \ + --title 'Mint-ordered log' \ + --summary-file "${{ github.workspace }}/world-sim-mint.md" \ + --detail-file "${{ github.workspace }}/world-sim-mint.txt" + + - name: Play the book (append-only log) + id: append-only + continue-on-error: true + run: | + pnpm --filter @workflow/sim-world-workbench sim \ + --no-color \ + --append-only \ + --title 'Append-only log' \ + --summary-file "${{ github.workspace }}/world-sim-append-only.md" \ + --detail-file "${{ github.workspace }}/world-sim-append-only.txt" + + # Four visible lines when collapsed: the heading, one line saying what + # this is, and one per world. Everything else is behind a fold. The + # comment is reposted on every push to the PR, so what it costs when it + # has nothing new to say is the thing to keep small. + - name: Render summary + if: always() + run: | + run_url="${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}" + { + echo "## Sim World" + echo + echo "Simulated world deterministic testing for races. [Traces]($run_url)" + echo + for world in mint append-only; do + if [ -f "world-sim-$world.md" ]; then + # The summary names its detail file by the path it was given, + # which is absolute so that pnpm's package-directory cwd cannot + # scatter them. Strip the workspace prefix back off, leaving + # the bare name the artifact below actually contains. + sed "s|${{ github.workspace }}/||g" "world-sim-$world.md" + # Blank line between the two folds. Adjacent HTML blocks with + # nothing between them get parsed as one, and the second world + # disappears into the first one's fold. + echo + else + echo "🟠 _The $world run produced no summary β€” see the job log._" + echo + fi + done + } | tee world-sim-summary.md >> "$GITHUB_STEP_SUMMARY" + + # Skipped on forks, where `pull_request` grants read-only permissions and + # the write would fail. The step summary above is the fallback there. + - name: Update PR comment + if: >- + always() && github.event_name == 'pull_request' && + github.event.pull_request.head.repo.full_name == github.repository + continue-on-error: true + uses: marocchino/sticky-pull-request-comment@773744901bac0e8cbb5a0dc842800d45e9b2b405 # v2.9.4 + with: + header: world-sim-results + path: world-sim-summary.md + + - name: Upload traces + if: always() + uses: actions/upload-artifact@v4 + with: + name: world-sim-traces + path: | + world-sim-summary.md + world-sim-mint.md + world-sim-mint.txt + world-sim-append-only.md + world-sim-append-only.txt + retention-days: 7 + if-no-files-found: ignore diff --git a/docs/content/docs/v5/configuration/runtime-tuning.mdx b/docs/content/docs/v5/configuration/runtime-tuning.mdx index 5811dd1ed9..53a59e7759 100644 --- a/docs/content/docs/v5/configuration/runtime-tuning.mdx +++ b/docs/content/docs/v5/configuration/runtime-tuning.mdx @@ -83,6 +83,14 @@ For example, a workflow can run a 10-minute inline step even with `WORKFLOW_REPL - As a result, 412 volume describes a workload rather than the health of a deployment, and a run that never sees one is not evidence the guard is inactive. - Set `0` to disable. +### `WORKFLOW_SLOT_GAP_CHECK` + +- Default: enabled +- A replay checks that the [event log](/docs/how-it-works/event-sourcing#event-ids) it loaded is dense before it runs, and fails the run with [`CORRUPTED_EVENT_LOG`](/docs/errors/corrupted-event-log) when a position below the log's highest holds no event. A log missing only its first position, meaning a run whose `run_created` is still being written, is left alone. +- A position can be briefly empty while the write that occupies it is still committing, so the check re-reads the log a few times before it decides, and the replay continues from whichever log it settled on. +- The check trades one failure for another. Most holes stand for an event that never happened, and replaying past those is correct. A hole standing for an event that did happen looks identical, and replaying past that one produces a run whose result is silently wrong. Failing is the recoverable side of that trade. +- Set `0` to replay across holes instead. + ### `WORKFLOW_PRECONDITION_MAX_INPROCESS_RESTARTS` - Default: `3` @@ -101,17 +109,6 @@ For example, a workflow can run a 10-minute inline step even with `WORKFLOW_REPL - Delay before a re-invocation caused by a rejected event creation. - Unlike an in-process restart, which re-reads immediately, a re-invocation only happens once the in-process budget failed to catch up β€” so the delay gives the other writers a moment to quiesce. -### `WORKFLOW_PER_KIND_CORRELATION_IDS` - -- Default: disabled -- Experimental. Gives each kind of entity a workflow creates β€” steps, waits, hooks, attribute writes, abort controllers, stream IDs β€” its own sequence of correlation IDs. -- With one sequence shared by every kind, an ID is an ordinal over the whole run, so a single extra draw of any kind shifts every ID after it. Two concurrent replays of the same run that disagree about one `sleep()` then assign different IDs to every step that follows, and each writes events the other can neither match nor consume, which fails the run with `CORRUPTED_EVENT_LOG`. Per-kind sequences confine that to the kind that actually differs. -- IDs remain ordered within a kind, so hooks created by your workflow are still listed in creation order. A hook the runtime creates for you, such as the one backing an abort controller, draws from its own kind and so is listed at an arbitrary position relative to your hooks rather than at its creation position. -- A run must replay under the scheme that minted its IDs. A replay that switches schemes mid-run assigns IDs its own earlier events do not carry, so it can consume none of them and the run fails. - - On Vercel, a run keeps replaying on the deployment it started on, so it only ever sees the value baked into that deployment. Changing the setting affects new runs only. - - Elsewhere β€” `@workflow/world-postgres`, `@workflow/world-local`, any self-hosted process β€” nothing pins a run to the code that started it. Turn the setting on during a quiet window with no runs in flight, and roll the new value out to your whole fleet at once: a rolling deploy that leaves both values live replays one run under two schemes concurrently, which is the failure the setting exists to reduce. -- Set `1` to enable. - ## Inline execution ### `WORKFLOW_V2_TIMEOUT_MS` diff --git a/docs/content/docs/v5/errors/corrupted-event-log.mdx b/docs/content/docs/v5/errors/corrupted-event-log.mdx index efd1028cde..f8d31bdb05 100644 --- a/docs/content/docs/v5/errors/corrupted-event-log.mdx +++ b/docs/content/docs/v5/errors/corrupted-event-log.mdx @@ -21,15 +21,18 @@ Workflow replay diverged times after reco ## Why This Happens -Workflows persist their progress as an ordered event log. During replay, the runtime processes each event in sequence β€” every event must be consumed by a matching callback (e.g., a step or sleep waiting for its result). When an event has no matching consumer, the runtime cannot advance past it, which would block all subsequent events and hang the workflow indefinitely. +Workflows persist their progress as an ordered event log. During replay, the runtime processes each event in sequence β€” every event must be consumed by a matching callback (e.g., a step or sleep waiting for its result). An event no callback ever claims is one the runtime would have to drop to finish the run, so it fails the run instead of returning a result that silently ignored it. -Instead of silently hanging, the runtime retries a divergent replay before failing the workflow and surfacing this terminal error. +A delivery written from outside the replay, such as a hook firing or a step completing on another invocation, can land ahead of the events the replay is writing itself. That is ordinary concurrency rather than corruption, so the runtime holds such an event and offers it to each consumer the replay registers afterwards. The failure comes only when the workflow function returns while an event is still held, at which point no consumer can ever appear. A replay that suspends still holding one reports it on the span (`workflow.events.parked.count`, `.event_id`, `.event_type`) and leaves the decision to the replay that follows. + +Before failing, the runtime retries a divergent replay and surfaces this terminal error only if replay still cannot recover. Common scenarios that produce this error: -1. **Duplicate completion events** β€” Two `wait_completed` events for a single `wait_created`, or two `step_completed` events for the same step. The first is consumed normally, but the second has no consumer. -2. **Orphaned events** β€” A `step_completed` or `wait_completed` event whose `correlationId` doesn't match any step or sleep in the workflow code. +1. **Duplicate completion events** β€” Two `wait_completed` events for a single `wait_created`, or two `step_completed` events for the same step. The first is consumed normally, and the second resolves something already resolved, so no later consumer can claim it. +2. **Orphaned events** β€” A `step_completed` or `wait_completed` event whose `correlationId` doesn't match any step or sleep in the workflow code, so the replay reaches its end still holding it. 3. **Events after terminal state** β€” An event that arrives after its corresponding step or wait has already reached a terminal state (e.g., `step_retrying` after `step_completed`). +4. **A hole in the log** β€” Events are numbered by their position in the run's log, and those positions are dense, so a position below the log's highest that holds no event means the log the replay loaded is incomplete. The runtime cannot tell a position no write ever occupied from one whose event it failed to read, so it refuses to replay rather than produce a result that may be silently wrong. See [`WORKFLOW_SLOT_GAP_CHECK`](/docs/configuration/runtime-tuning#workflow_slot_gap_check). ## What To Do diff --git a/docs/content/docs/v5/how-it-works/event-sourcing.mdx b/docs/content/docs/v5/how-it-works/event-sourcing.mdx index 70ae0baf5c..31febb849a 100644 --- a/docs/content/docs/v5/how-it-works/event-sourcing.mdx +++ b/docs/content/docs/v5/how-it-works/event-sourcing.mdx @@ -260,7 +260,7 @@ On Vercel, `requestId` is the platform request ID when available. Other worlds a ## Entity IDs -All entities in the Workflow SDK use a consistent ID format: a 4-character prefix followed by an underscore and a [ULID](https://github.com/ulid/spec) (Universally Unique Lexicographically Sortable Identifier). +All entities in the Workflow SDK use a consistent ID format: a 4-character prefix followed by an underscore and a fixed-width body. For every entity except events, that body is a [ULID](https://github.com/ulid/spec) (Universally Unique Lexicographically Sortable Identifier). An event's body is its slot number, described below. | Entity | Prefix | Example | |--------|--------|---------| @@ -268,11 +268,19 @@ All entities in the Workflow SDK use a consistent ID format: a 4-character prefi | Step | `step_` | `step_01HXYZ123ABC456DEF789GHJ` | | Hook | `hook_` | `hook_01HXYZ123ABC456DEF789GHJ` | | Wait | `wait_` | `wait_01HXYZ123ABC456DEF789GHJ` | -| Event | `evnt_` | `evnt_01HXYZ123ABC456DEF789GHJ` | +| Event | `evnt_` | `evnt_00000000000000000000000042` (slot 42) | | Stream | `strm_` | `strm_01HXYZ123ABC456DEF789GHJ` | **Why this format?** - **Prefixes enable introspection**: Given any ID, you can immediately identify what type of entity it refers to. This makes debugging, logging, and cross-referencing entities across the system straightforward. -- **ULIDs enable chronological ordering**: Unlike UUIDs, ULIDs encode a timestamp in their first 48 bits, making them lexicographically sortable by creation time. This property is essential for the event logβ€”events are always stored and retrieved in the correct chronological order simply by sorting their IDs. +- **Fixed-width bodies enable ordering**: Unlike UUIDs, these bodies sort lexicographically in creation order, so the event log is stored and retrieved in the correct order by sorting IDs alone. Slot numbers get that from counting at a fixed width, which makes string order the same as numeric order. ULIDs get it from the timestamp in their first 48 bits, which also makes a ULID's creation time recoverable from the ID itself. + +### Event IDs + +An event ID is a **slot number**: the event's 1-based position in the run's event log, zero-padded to the same width as a ULID. The world assigns it when the event is published, so two writers racing to append never claim the same position and a rejected write leaves no gap behind. Slots are dense, and unique only within a run, so an event ID identifies an event only when paired with its `runId`. + +Density is what lets a reader tell a complete log from an incomplete one by its length alone. A replay that loads a log with a position missing below the highest one it can see cannot tell an event that was never written from one it failed to read, so it fails the run with [`CORRUPTED_EVENT_LOG`](/docs/errors/corrupted-event-log) rather than replay across the hole. See [`WORKFLOW_SLOT_GAP_CHECK`](/docs/configuration/runtime-tuning#workflow_slot_gap_check). + +A slot ID carries no timestamp. Zero-padded decimal digits are a subset of the ULID alphabet, so a slot ID passes ULID validation and sorts correctly, but decoding its first 48 bits yields the Unix epoch instead of a creation time. Read `createdAt` on the event when you need to know when it was written, and don't decode the time from an `evnt_` ID you get back from an API, a log line, or a cursor. diff --git a/docs/content/worlds/v5/building-a-world.mdx b/docs/content/worlds/v5/building-a-world.mdx index 619fad3fc9..33062c2044 100644 --- a/docs/content/worlds/v5/building-a-world.mdx +++ b/docs/content/worlds/v5/building-a-world.mdx @@ -36,6 +36,8 @@ interface WorldCapabilities { hookRetention?: { active: boolean; }; + slotEventIds?: boolean; + preconditionGuard?: boolean; } interface World extends Storage, Queue, Streamer { @@ -47,7 +49,7 @@ interface World extends Storage, Queue, Streamer { } ``` -The optional `capabilities` object advertises additional behavior. Set `hookRetention.active` to `true` only when the World implements Hook token retention. The optional `start()` method initializes background tasks (for example, queue polling). The optional `close()` method releases resources like connection pools and listeners. The optional `getEncryptionKeyForRun()` method returns the AES-256 key used to encrypt data for a run; if it is not implemented, encryption is disabled. +The optional `capabilities` object advertises additional behavior. Set `hookRetention.active` to `true` only when the World implements Hook token retention, `slotEventIds` when it allocates [slot-numbered event IDs](#event-id-allocation), and `preconditionGuard` when it enforces the [precondition guard](#optional-the-event-creation-precondition-guard). The optional `start()` method initializes background tasks (for example, queue polling). The optional `close()` method releases resources like connection pools and listeners. The optional `getEncryptionKeyForRun()` method returns the AES-256 key used to encrypt data for a run; if it is not implemented, encryption is disabled. ## The Event Log Model @@ -106,6 +108,19 @@ Keep the owning Run available for at least as long as its token remains unavaila **Automatic Hook Cleanup:** When a run ends, remove its live Hooks. Make each token available unless its `tokenRetentionUntil` is still in the future. A `hook_disposed` event always makes the token available immediately. +### Event ID Allocation + +Your World assigns every event ID. An ID is `evnt_` followed by the event's 1-based position in that run's log, zero-padded to 26 characters, so the run's first event is `evnt_00000000000000000000000001`. Use `slotToEventId()` from `@workflow/world` to format one, and declare `capabilities.slotEventIds`. + +Two properties have to hold, and both are about what a reader can conclude from the log: + +- **Uniqueness.** Two writers racing to append must not both take a position. Settle it where the store settles it, with a unique constraint on `(runId, eventId)` or a conditional write, rather than reading the maximum and adding one in your own process. +- **Density.** Positions run from 1 with no holes, which is what lets a reader tell a complete log from a truncated one by its length alone. A writer that loses a race must re-derive its position from the store and take the next free one. Incrementing a local number after a loss leaves a permanent hole, and the runtime treats a hole as a log it cannot safely replay across. + +`events.create()` params carry `eventCount`: how many events the writer held in the log it replayed from, which is the position it expects to land on minus one. Attempt `eventCount + 1`. When that position is taken, **do not reject the write**. Advance to the next free position, commit there, and return the events occupying the positions you skipped over on the success response, in `events` with a matching `cursor` and `hasMore`. The writer merges them into its own log and replays once, rather than paying a second round trip to discover it was behind. A caller with a stale count is the normal case for a fan-out, and rejecting it would serialize writes the runtime issues in parallel. + +`eventCount` supersedes the `stateUpdatedAt` / `stateEventCount` / `stateCursor` triple below for a World that allocates positions. The triple approximates a position with a timestamp watermark plus a count of events at or below it, which a complete-but-stale snapshot passes: every event the writer holds is at or below its own watermark, so the count matches and no fence fires. A dense position has no such blind spot. + ### Optional: The Event Creation Precondition Guard A replay writes events derived from the event log it loaded, so a write made from an event log that no longer matches the store can commit events no correct replay would produce. To let a World fence those writes, `events.create()` params may carry a description of the snapshot the caller replayed from: diff --git a/docs/content/worlds/v5/postgres.mdx b/docs/content/worlds/v5/postgres.mdx index a9d3c5172e..82757b83fd 100644 --- a/docs/content/worlds/v5/postgres.mdx +++ b/docs/content/worlds/v5/postgres.mdx @@ -27,8 +27,8 @@ Use the same release channel for `workflow` and `@workflow/world-postgres`. If your app uses a beta or other prerelease Workflow version, install the matching prerelease Postgres World package, such as `npm install @workflow/world-postgres@beta`. Mismatched versions fail before -starting a run with an error that says the runtime requires a World with a -matching spec version. +starting a run with an error that names the spec versions the runtime supports +and the one the World declares. Configure the required environment variables to use the world and point it to your PostgreSQL database: diff --git a/packages/core/src/abort-consistency.test.ts b/packages/core/src/abort-consistency.test.ts index d99b67de1d..ec93e93227 100644 --- a/packages/core/src/abort-consistency.test.ts +++ b/packages/core/src/abort-consistency.test.ts @@ -11,10 +11,6 @@ import type { Event, WorkflowRun } from '@workflow/world'; import * as nanoid from 'nanoid'; import { monotonicFactory } from 'ulid'; import { describe, expect, it, vi } from 'vitest'; -import { - createCorrelationIdGenerator, - isPerKindCorrelationIdsEnabled, -} from './correlation-id.js'; import { EventsConsumer } from './events-consumer.js'; import type { WorkflowSuspension } from './global.js'; import type { WorkflowOrchestratorContext } from './private.js'; @@ -44,16 +40,13 @@ function setupWorkflowContext(events: Event[]): WorkflowOrchestratorContext { replayPayloadCache: new ReplayPayloadCache(undefined), globalThis: context.globalThis, eventsConsumer: new EventsConsumer(events, { + // Fake context: no deliveries are modeled, so the gate is a no-op here. + isDeliveryIdle: () => true, onUnconsumedEvent: () => {}, getPromiseQueue: () => Promise.resolve(), }), invocationsQueue: new Map(), - generateCorrelationId: createCorrelationIdGenerator({ - seed: 'test', - fixedTimestamp: workflowStartedAt, - positional: () => ulid(workflowStartedAt), - perKind: isPerKindCorrelationIdsEnabled(), - }), + generateUlid: () => ulid(workflowStartedAt), generateNanoid: nanoid.customRandom(nanoid.urlAlphabet, 21, (size) => new Uint8Array(size).map(() => 256 * context.globalThis.Math.random()) ), diff --git a/packages/core/src/abort-controller.test.ts b/packages/core/src/abort-controller.test.ts index 2ced0b4aca..3adf469366 100644 --- a/packages/core/src/abort-controller.test.ts +++ b/packages/core/src/abort-controller.test.ts @@ -12,10 +12,6 @@ import type { Event } from '@workflow/world'; import * as nanoid from 'nanoid'; import { monotonicFactory } from 'ulid'; import { describe, expect, it, vi } from 'vitest'; -import { - createCorrelationIdGenerator, - isPerKindCorrelationIdsEnabled, -} from './correlation-id.js'; import { DEFERRED_CHECK_DELAY_MS, EventsConsumer } from './events-consumer.js'; import type { WorkflowOrchestratorContext } from './private.js'; import { ReplayPayloadCache } from './replay-payload-cache.js'; @@ -42,16 +38,13 @@ function setupWorkflowContext( replayPayloadCache: new ReplayPayloadCache(undefined), globalThis: context.globalThis, eventsConsumer: new EventsConsumer(events, { + // Fake context: no deliveries are modeled, so the gate is a no-op here. + isDeliveryIdle: () => true, onUnconsumedEvent, getPromiseQueue: () => ctx.promiseQueue, }), invocationsQueue: new Map(), - generateCorrelationId: createCorrelationIdGenerator({ - seed: 'test', - fixedTimestamp: workflowStartedAt, - positional: () => ulid(workflowStartedAt), - perKind: isPerKindCorrelationIdsEnabled(), - }), + generateUlid: () => ulid(workflowStartedAt), generateNanoid: nanoid.customRandom(nanoid.urlAlphabet, 21, (size) => new Uint8Array(size).map(() => 256 * context.globalThis.Math.random()) ), diff --git a/packages/core/src/abort-replay-ordering.test.ts b/packages/core/src/abort-replay-ordering.test.ts index 8e7a1a9e98..8961a64925 100644 --- a/packages/core/src/abort-replay-ordering.test.ts +++ b/packages/core/src/abort-replay-ordering.test.ts @@ -27,10 +27,6 @@ import type { Event } from '@workflow/world'; import * as nanoid from 'nanoid'; import { monotonicFactory } from 'ulid'; import { describe, expect, it, vi } from 'vitest'; -import { - createCorrelationIdGenerator, - isPerKindCorrelationIdsEnabled, -} from './correlation-id.js'; import { EventsConsumer } from './events-consumer.js'; import { scheduleWhenIdle, @@ -77,16 +73,13 @@ function setupWorkflowContext(events: Event[]): WorkflowOrchestratorContext { replayPayloadCache: new ReplayPayloadCache(undefined), globalThis: context.globalThis, eventsConsumer: new EventsConsumer(events, { + // Fake context: no deliveries are modeled, so the gate is a no-op here. + isDeliveryIdle: () => true, onUnconsumedEvent: () => {}, getPromiseQueue: () => ctx.promiseQueue, }), invocationsQueue: new Map(), - generateCorrelationId: createCorrelationIdGenerator({ - seed: 'test', - fixedTimestamp: workflowStartedAt, - positional: () => ulid(workflowStartedAt), - perKind: isPerKindCorrelationIdsEnabled(), - }), + generateUlid: () => ulid(workflowStartedAt), generateNanoid: nanoid.customRandom(nanoid.urlAlphabet, 21, (size) => new Uint8Array(size).map(() => 256 * context.globalThis.Math.random()) ), diff --git a/packages/core/src/async-deserialization-ordering.test.ts b/packages/core/src/async-deserialization-ordering.test.ts index 4ad65f662c..41d4684563 100644 --- a/packages/core/src/async-deserialization-ordering.test.ts +++ b/packages/core/src/async-deserialization-ordering.test.ts @@ -4,7 +4,6 @@ import * as nanoid from 'nanoid'; import { monotonicFactory } from 'ulid'; import { afterEach, beforeAll, describe, expect, it, vi } from 'vitest'; import { registerSerializationClass } from './class-serialization.js'; -import { createCorrelationIdGenerator } from './correlation-id.js'; import { EventsConsumer } from './events-consumer.js'; import type { WorkflowOrchestratorContext } from './private.js'; import { ReplayPayloadCache } from './replay-payload-cache.js'; @@ -54,18 +53,13 @@ function setupWorkflowContext(events: Event[]): WorkflowOrchestratorContext { replayPayloadCache: new ReplayPayloadCache(undefined), globalThis: context.globalThis, eventsConsumer: new EventsConsumer(events, { + // Fake context: no deliveries are modeled, so the gate is a no-op here. + isDeliveryIdle: () => true, onUnconsumedEvent: () => {}, getPromiseQueue: () => Promise.resolve(), }), invocationsQueue: new Map(), - generateCorrelationId: createCorrelationIdGenerator({ - seed: 'test', - fixedTimestamp: workflowStartedAt, - positional: () => ulid(workflowStartedAt), - // The event logs in this file hardcode correlation ids the run-wide - // shared sequence minted, so replay only matches under that scheme. - perKind: false, - }), + generateUlid: () => ulid(workflowStartedAt), generateNanoid: nanoid.customRandom(nanoid.urlAlphabet, 21, (size) => new Uint8Array(size).map(() => 256 * context.globalThis.Math.random()) ), diff --git a/packages/core/src/correlation-id-replay.test.ts b/packages/core/src/correlation-id-replay.test.ts deleted file mode 100644 index 92b1db81b6..0000000000 --- a/packages/core/src/correlation-id-replay.test.ts +++ /dev/null @@ -1,124 +0,0 @@ -import type { Event } from '@workflow/world'; -import * as nanoid from 'nanoid'; -import { monotonicFactory } from 'ulid'; -import { describe, expect, it, vi } from 'vitest'; -import { createCorrelationIdGenerator } from './correlation-id.js'; -import { EventsConsumer } from './events-consumer.js'; -import type { WorkflowOrchestratorContext } from './private.js'; -import { ReplayPayloadCache } from './replay-payload-cache.js'; -import { dehydrateStepReturnValue } from './serialization.js'; -import { createUseStep } from './step.js'; -import { createContext } from './vm/index.js'; -import { createCreateHook } from './workflow/hook.js'; -import { createSleep } from './workflow/sleep.js'; - -/** - * Correlation-id stability seen through the primitives that actually mint ids, - * rather than through the generator alone: that a step's id survives another - * kind of entity being created alongside it, and that a replay consumes an event - * log carrying the ids a same-seeded replay derives. - * - * The rest of the replay suites author their event logs with literal correlation - * ids from the shared sequence and pin themselves to it. These fixtures derive - * their ids instead, so they hold under either scheme. - */ - -const SEED = 'test'; -const FIXED_TIMESTAMP = 1753481739458; - -function setupWorkflowContext( - events: Event[], - perKind: boolean -): WorkflowOrchestratorContext { - const context = createContext({ - seed: SEED, - fixedTimestamp: FIXED_TIMESTAMP, - }); - const ulid = monotonicFactory(() => context.globalThis.Math.random()); - return { - runId: 'wrun_test', - encryptionKey: undefined, - replayPayloadCache: new ReplayPayloadCache(undefined), - globalThis: context.globalThis, - eventsConsumer: new EventsConsumer(events, { - onUnconsumedEvent: () => {}, - getPromiseQueue: () => Promise.resolve(), - }), - invocationsQueue: new Map(), - generateCorrelationId: createCorrelationIdGenerator({ - seed: SEED, - fixedTimestamp: FIXED_TIMESTAMP, - positional: () => ulid(FIXED_TIMESTAMP), - perKind, - }), - generateNanoid: nanoid.customRandom(nanoid.urlAlphabet, 21, (size) => - new Uint8Array(size).map(() => 256 * context.globalThis.Math.random()) - ), - onWorkflowError: vi.fn(), - promiseQueue: Promise.resolve(), - pendingDeliveries: 0, - pendingDeliveryBarriers: new Map(), - }; -} - -/** - * The id the next step of a replay would claim. Nothing in the log resolves the - * step, so the returned promise stays pending by design: the queue item is what - * we are after. - */ -function probeStepId( - perKind: boolean, - before?: (ctx: WorkflowOrchestratorContext) => void -): string { - const ctx = setupWorkflowContext([], perKind); - before?.(ctx); - void createUseStep(ctx)('add')(1, 2).catch(() => {}); - const item = [...ctx.invocationsQueue.values()].find( - (entry) => entry.type === 'step' - ); - if (!item) { - throw new Error('expected a step invocation'); - } - return item.correlationId; -} - -function createHookAndSleep(ctx: WorkflowOrchestratorContext): void { - createCreateHook(ctx)(); - void createSleep(ctx)('1h').catch(() => {}); -} - -describe('correlation ids through the replay primitives', () => { - it('keeps a step id when a hook and a sleep are created before it', () => { - expect(probeStepId(true, createHookAndSleep)).toBe(probeStepId(true)); - }); - - it('renumbers that step under one sequence shared by every kind', () => { - // The failure this PR removes, and the reason the assertion above is worth - // making: with a shared sequence the hook and the sleep consume the two - // ordinals the step would otherwise have drawn from. - expect(probeStepId(false, createHookAndSleep)).not.toBe(probeStepId(false)); - }); - - it('consumes a step_completed authored with the derived id', async () => { - const correlationId = probeStepId(true, createHookAndSleep); - const ctx = setupWorkflowContext( - [ - { - eventId: 'evnt_0', - runId: 'wrun_test', - eventType: 'step_completed', - correlationId, - eventData: { - stepName: 'add', - result: await dehydrateStepReturnValue(3, 'wrun_test', undefined), - }, - createdAt: new Date(), - }, - ], - true - ); - createHookAndSleep(ctx); - await expect(createUseStep(ctx)('add')(1, 2)).resolves.toBe(3); - expect(ctx.onWorkflowError).not.toHaveBeenCalled(); - }); -}); diff --git a/packages/core/src/correlation-id.test.ts b/packages/core/src/correlation-id.test.ts deleted file mode 100644 index db95c65df4..0000000000 --- a/packages/core/src/correlation-id.test.ts +++ /dev/null @@ -1,154 +0,0 @@ -import { decodeTime, monotonicFactory } from 'ulid'; -import { describe, expect, it } from 'vitest'; -import { - CORRELATION_ID_LENGTH, - type CorrelationIdKind, - createCorrelationIdGenerator, - isPerKindCorrelationIdsEnabled, -} from './correlation-id.js'; - -const SEED = 'wrun_abc:myWorkflow:dpl_123'; -const FIXED_TIMESTAMP = 1753481739458; - -function makeGenerator( - overrides: { seed?: string; fixedTimestamp?: number; perKind?: boolean } = {} -) { - // A stand-in for the run's shared sequence. Seeded so the positional mode is - // reproducible across the two generators a replay-stability test builds. - let counter = 0; - const ulid = monotonicFactory(() => { - counter = (counter * 1103515245 + 12345) % 2147483648; - return counter / 2147483648; - }); - const fixedTimestamp = overrides.fixedTimestamp ?? FIXED_TIMESTAMP; - return createCorrelationIdGenerator({ - seed: overrides.seed ?? SEED, - fixedTimestamp, - positional: () => ulid(fixedTimestamp), - perKind: overrides.perKind ?? true, - }); -} - -const KINDS: CorrelationIdKind[] = [ - 'step', - 'wait', - 'hook', - 'attr', - 'abort', - 'abortHook', - 'stream', -]; - -describe('createCorrelationIdGenerator', () => { - it('mints syntactically valid ULIDs carrying fixedTimestamp', () => { - const generate = makeGenerator(); - for (const kind of KINDS) { - const id = generate(kind); - expect(id).toHaveLength(CORRELATION_ID_LENGTH); - expect(id).toMatch(/^[0-9A-HJKMNP-TV-Z]+$/); - expect(decodeTime(id)).toBe(FIXED_TIMESTAMP); - } - }); - - it('is deterministic across replays of the same run', () => { - const first = makeGenerator(); - const second = makeGenerator(); - const draw = (generate: (kind: CorrelationIdKind) => string) => [ - generate('step'), - generate('step'), - generate('wait'), - generate('step'), - generate('hook'), - ]; - expect(draw(first)).toEqual(draw(second)); - }); - - it('mints different ids for different runs', () => { - const first = makeGenerator({ seed: 'wrun_one:w:dpl' }); - const second = makeGenerator({ seed: 'wrun_two:w:dpl' }); - expect(first('step')).not.toBe(second('step')); - }); - - it('gives every kind its own starting point', () => { - const generate = makeGenerator(); - const ids = KINDS.map((kind) => generate(kind)); - expect(new Set(ids).size).toBe(KINDS.length); - }); - - it('increases monotonically within a kind', () => { - const generate = makeGenerator(); - const ids = [generate('hook'), generate('hook'), generate('hook')]; - expect(ids).toEqual([...ids].sort()); - expect(new Set(ids).size).toBe(ids.length); - }); - - it('does not renumber one kind when another draws more often', () => { - // The whole point of per-kind sources: two replays that disagree about how - // many hooks, sleeps or streams were created still agree about which id - // belongs to the Nth step. - const withoutExtras = makeGenerator(); - const withExtras = makeGenerator(); - - const steps = [withoutExtras('step'), withoutExtras('step')]; - - withExtras('hook'); - const interleaved = [withExtras('step')]; - withExtras('wait'); - withExtras('stream'); - withExtras('attr'); - withExtras('abort'); - interleaved.push(withExtras('step')); - - expect(interleaved).toEqual(steps); - }); - - it('keeps abort controllers from renumbering user hooks', () => { - const withoutController = makeGenerator(); - const withController = makeGenerator(); - withController('abort'); - withController('abortHook'); - expect(withController('hook')).toBe(withoutController('hook')); - }); - - it('keeps every id on fixedTimestamp in both modes', () => { - // `monotonicFactory` returns `encodeTime(lastTime)` on its increment branch, - // so a single draw that omits the seed time latches the host wall clock and - // every later id in the run carries a timestamp that differs per replay. - // Stream ids used to be drawn that way. - for (const perKind of [true, false]) { - const generate = makeGenerator({ perKind }); - for (const kind of ['stream', 'stream', 'step', 'hook'] as const) { - expect(decodeTime(generate(kind))).toBe(FIXED_TIMESTAMP); - } - } - }); - - it('ignores the kind when per-kind sources are disabled', () => { - const generate = makeGenerator({ perKind: false }); - const shared = makeGenerator({ perKind: false }); - // Positional mode is one sequence for the whole run, so drawing `wait` - // consumes the ordinal the next `step` would otherwise have had. - expect(generate('step')).toBe(shared('step')); - expect(generate('wait')).toBe(shared('step')); - }); -}); - -describe('isPerKindCorrelationIdsEnabled', () => { - it('reads WORKFLOW_PER_KIND_CORRELATION_IDS, defaulting to disabled', () => { - const original = process.env.WORKFLOW_PER_KIND_CORRELATION_IDS; - try { - delete process.env.WORKFLOW_PER_KIND_CORRELATION_IDS; - expect(isPerKindCorrelationIdsEnabled()).toBe(false); - process.env.WORKFLOW_PER_KIND_CORRELATION_IDS = '1'; - expect(isPerKindCorrelationIdsEnabled()).toBe(true); - process.env.WORKFLOW_PER_KIND_CORRELATION_IDS = '0'; - expect(isPerKindCorrelationIdsEnabled()).toBe(false); - } finally { - if (original === undefined) { - delete process.env.WORKFLOW_PER_KIND_CORRELATION_IDS; - } else { - process.env.WORKFLOW_PER_KIND_CORRELATION_IDS = original; - } - } - }); -}); diff --git a/packages/core/src/correlation-id.ts b/packages/core/src/correlation-id.ts deleted file mode 100644 index dbed6f8f5c..0000000000 --- a/packages/core/src/correlation-id.ts +++ /dev/null @@ -1,234 +0,0 @@ -import { encodeTime, incrementBase32 } from 'ulid'; - -/** - * Correlation-id generation for the entity families a replay can create. - * - * Correlation ids are minted by the workflow VM and are the server's identity - * gate: a conditional create on the id is what makes a duplicate write from a - * second live replay idempotent instead of additive. That only works if two - * replays of the same run mint the same id for the same entity. - * - * Historically every id was the Nth draw of *one* monotonic ULID sequence per - * run, shared by steps, waits, hooks, attribute writes, abort controllers and - * stream ids alike. Every id was therefore an ordinal over the whole run, and a - * single extra draw of any kind renumbered every entity of every kind after it. - * Two replays that agreed about every step but disagreed about one `sleep()` - * would mint different ids for all subsequent steps, so their writes appended - * side by side instead of colliding, and the settled log ended up holding two - * names for one logical step. Only one of them can be consumed on the next - * replay; the other is fatal (`onUnconsumedEvent`). - * - * Per-kind sources narrow that coupling to one kind at a time: each family - * draws from its own independent incrementing sequence, so a disagreement about - * how many hooks or sleeps were created no longer renames steps. - * - * Ids stay syntactically valid ULIDs (10 Crockford characters of - * `fixedTimestamp` plus 16 of body), because correlation ids are validated as - * prefixed 26-char ULIDs by the backend, and they stay monotonic *within* a - * kind, because `hooks.list` is ordered by hook id. - * - * Monotonicity is per kind, and two kinds mint `hook_` ids (`hook` and - * `abortHook`), so listing order is only creation order *within* each of them. - * No world filters system hooks out of a listing, so a run that constructs an - * abort controller and also creates its own hooks lists that system hook at a - * position decided by its kind's hash rather than at its creation position. - * Order among the user's own hooks is unaffected. - * - * This does not make ids independent of *ordinal position within their own - * kind*: two replays that disagree about how many steps ran still mint - * different ids for the next step. That is a narrower failure than the shared - * sequence's, not an eliminated one. - */ - -/** Entity families that draw correlation ids, each from its own sequence. */ -export type CorrelationIdKind = - /** `step_` ids, one per step invocation. */ - | 'step' - /** `wait_` ids, one per `sleep()`. */ - | 'wait' - /** `hook_` ids for hooks created by workflow code. */ - | 'hook' - /** `attr_` ids, one per attribute write. */ - | 'attr' - /** - * The abort controller's own id, which becomes its stream name and hook - * token. Separate from `hook` so constructing an abort controller does not - * renumber later user hooks. - */ - | 'abort' - /** `hook_` ids for the internal system hook backing an abort controller. */ - | 'abortHook' - /** - * Ids minted during serialization (`STABLE_ULID`): stream names, and an abort - * holder's stream name and `abrt_` hook token when it reaches serialization - * without an identity yet (`reduceAbortWithListener`), which is why `abort` - * above is not the only mint path for an abort identity. Not correlation ids, - * but they drew from the same shared sequence, so a workflow that serialized - * a stream renumbered every entity created after it. - */ - | 'stream'; - -/** Mints the ULID body of a correlation id for one entity family. */ -export type CorrelationIdGenerator = (kind: CorrelationIdKind) => string; - -const CROCKFORD = '0123456789ABCDEFGHJKMNPQRSTVWXYZ'; - -/** Number of Crockford characters in a ULID's random component. */ -const BODY_CHARS = 16; - -/** Number of Crockford characters in a ULID's timestamp component. */ -const TIME_CHARS = 10; - -function mul32(a: number, b: number): number { - return Math.imul(a, b) >>> 0; -} - -function rotl32(value: number, shift: number): number { - return ((value << shift) | (value >>> (32 - shift))) >>> 0; -} - -/** MurmurHash3's 32-bit finalizer. */ -function fmix32(input: number): number { - let h = input >>> 0; - h = (h ^ (h >>> 16)) >>> 0; - h = mul32(h, 0x85ebca6b); - h = (h ^ (h >>> 13)) >>> 0; - h = mul32(h, 0xc2b2ae35); - return (h ^ (h >>> 16)) >>> 0; -} - -/** - * Deterministic 128-bit hash of a string, as four 32-bit lanes. - * - * A MurmurHash3-style mixer over UTF-16 code units, rotating which lane absorbs - * each unit and diffusing across lanes at the end. Only determinism and - * diffusion matter here: this is not a cryptographic hash, claims no - * bit-compatibility with any reference implementation, and must not be used for - * anything that outlives a deployment's replays. - */ -function hash128(input: string): [number, number, number, number] { - const lanes: [number, number, number, number] = [ - 0x9e3779b1, 0x85ebca77, 0xc2b2ae3d, 0x27d4eb2f, - ]; - for (let i = 0; i < input.length; i++) { - let k = input.charCodeAt(i) >>> 0; - k = mul32(k, 0xcc9e2d51); - k = rotl32(k, 15); - k = mul32(k, 0x1b873593); - const lane = i & 3; - let h = (lanes[lane] ^ k) >>> 0; - h = rotl32(h, 13); - lanes[lane] = (mul32(h, 5) + 0xe6546b64) >>> 0; - } - lanes[0] = (lanes[0] ^ input.length) >>> 0; - // Two passes so every lane depends on every other lane. - for (let pass = 0; pass < 2; pass++) { - for (let lane = 0; lane < 4; lane++) { - const previous = lanes[(lane + 3) & 3]; - lanes[lane] = fmix32((lanes[lane] ^ previous) >>> 0); - } - } - return lanes; -} - -/** - * Derives a kind's starting body: 80 bits of a 128-bit hash as 16 Crockford - * characters, most significant first. - * - * The leading character is confined to the alphabet's lower half so the body - * starts below half of the 80-bit space. `incrementBase32` throws on overflow, - * and without this a base that happened to land near `Z…Z` would make overflow - * reachable after few draws rather than after 2^79 of them. - */ -function deriveBody(seed: string, kind: CorrelationIdKind): string { - const lanes = hash128(`${seed} correlation-kind ${kind}`); - const bytes = [ - (lanes[0] >>> 24) & 0xff, - (lanes[0] >>> 16) & 0xff, - (lanes[0] >>> 8) & 0xff, - lanes[0] & 0xff, - (lanes[1] >>> 24) & 0xff, - (lanes[1] >>> 16) & 0xff, - (lanes[1] >>> 8) & 0xff, - lanes[1] & 0xff, - (lanes[2] >>> 24) & 0xff, - (lanes[2] >>> 16) & 0xff, - ]; - let body = ''; - let accumulator = 0; - let bits = 0; - for (const byte of bytes) { - accumulator = ((accumulator << 8) | byte) >>> 0; - bits += 8; - while (bits >= 5) { - const index = (accumulator >>> (bits - 5)) & 31; - body += CROCKFORD[body.length === 0 ? index & 15 : index]; - bits -= 5; - } - } - return body; -} - -/** - * Builds a replay's correlation-id generator. - * - * `perKind: false` returns the run's single shared monotonic sequence and - * ignores the kind entirely, so both schemes go through one call path and the - * flag is the only difference between them. - */ -export function createCorrelationIdGenerator(options: { - /** - * The run's replay-stable seed. Must not vary between replays of one run, and - * must differ between runs, or two runs would mint identical ids. - */ - seed: string; - fixedTimestamp: number; - /** The run's shared monotonic sequence, used as-is when `perKind` is false. */ - positional: () => string; - perKind: boolean; -}): CorrelationIdGenerator { - const { seed, fixedTimestamp, positional, perKind } = options; - - if (!perKind) { - return positional; - } - - const time = encodeTime(fixedTimestamp, TIME_CHARS); - const bodies = new Map(); - - return (kind: CorrelationIdKind) => { - const previous = bodies.get(kind); - const body = - previous === undefined - ? deriveBody(seed, kind) - : incrementBase32(previous); - bodies.set(kind, body); - return `${time}${body}`; - }; -} - -/** Length of a ULID, exported so tests need not restate it. */ -export const CORRELATION_ID_LENGTH = TIME_CHARS + BODY_CHARS; - -/** - * Whether each entity family draws correlation ids from its own sequence rather - * than from one sequence shared by the whole run. Off unless opted in, so an SDK - * upgrade alone never moves a run between schemes. - * - * The invariant either way: a run must replay under the scheme that minted its - * ids. A replay under the other scheme mints ids its own earlier events do not - * carry, so it can consume none of them and fails the run. Two things can break - * it, and both are about turning the flag on rather than about upgrading: - * - * - Enabling it while runs are in flight. On Vercel, skew protection keeps a run - * on the deployment that started it, so a run only ever sees the value baked - * into its own deployment. Elsewhere (world-postgres, world-local, a - * self-hosted process) nothing pins a run to the code that started it, so - * enable it during a quiet window. - * - A rolling deploy that leaves both values live, which puts two schemes on one - * run concurrently β€” the side-by-side append this whole mechanism exists to - * avoid. Roll the value out to the whole fleet at once. - */ -export function isPerKindCorrelationIdsEnabled(): boolean { - return process.env.WORKFLOW_PER_KIND_CORRELATION_IDS === '1'; -} diff --git a/packages/core/src/delivery-barrier-coverage.test.ts b/packages/core/src/delivery-barrier-coverage.test.ts index b37a8f597d..c40fb888cf 100644 --- a/packages/core/src/delivery-barrier-coverage.test.ts +++ b/packages/core/src/delivery-barrier-coverage.test.ts @@ -51,7 +51,6 @@ import type { Event } from '@workflow/world'; import * as nanoid from 'nanoid'; import { monotonicFactory } from 'ulid'; import { describe, expect, it, vi } from 'vitest'; -import { createCorrelationIdGenerator } from './correlation-id.js'; import { EventsConsumer } from './events-consumer.js'; import { WorkflowSuspension } from './global.js'; import { @@ -86,6 +85,8 @@ function setupWorkflowContext(events: Event[]): WorkflowOrchestratorContext { replayPayloadCache: new ReplayPayloadCache(undefined), globalThis: context.globalThis, eventsConsumer: new EventsConsumer(events, { + // Fake context: no deliveries are modeled, so the gate is a no-op here. + isDeliveryIdle: () => true, onUnconsumedEvent: (event) => { ctxRef.current?.onWorkflowError( new WorkflowRuntimeError(`Unconsumed event: ${event.eventType}`) @@ -94,14 +95,7 @@ function setupWorkflowContext(events: Event[]): WorkflowOrchestratorContext { getPromiseQueue: () => promiseQueueHolder.current, }), invocationsQueue: new Map(), - generateCorrelationId: createCorrelationIdGenerator({ - seed: 'test', - fixedTimestamp: workflowStartedAt, - positional: () => ulid(workflowStartedAt), - // The event logs in this file hardcode correlation ids the run-wide - // shared sequence minted, so replay only matches under that scheme. - perKind: false, - }), + generateUlid: () => ulid(workflowStartedAt), generateNanoid: nanoid.customRandom(nanoid.urlAlphabet, 21, (size) => new Uint8Array(size).map(() => 256 * context.globalThis.Math.random()) ), diff --git a/packages/core/src/events-consumer.test.ts b/packages/core/src/events-consumer.test.ts index c85a660ea6..9bae822106 100644 --- a/packages/core/src/events-consumer.test.ts +++ b/packages/core/src/events-consumer.test.ts @@ -21,9 +21,12 @@ function createMockEvent(overrides: Partial = {}): Event { } // Default options for tests that don't care about onUnconsumedEvent +// No deliveries are modeled here, so the delivery-idle gate is always open; the +// tests that exercise the gate itself pass their own predicate. const defaultOptions = { onUnconsumedEvent: vi.fn(), getPromiseQueue: () => Promise.resolve(), + isDeliveryIdle: () => true, }; // Helper function to wait for next tick @@ -165,6 +168,7 @@ describe('EventsConsumer', () => { const consumer = new EventsConsumer([event], { onUnconsumedEvent: unconsumedReceived.resolve, getPromiseQueue: () => Promise.resolve(), + isDeliveryIdle: () => true, }); const callback1 = vi .fn() @@ -421,6 +425,7 @@ describe('EventsConsumer', () => { const consumer = new EventsConsumer([event], { onUnconsumedEvent: unconsumedReceived.resolve, getPromiseQueue: () => Promise.resolve(), + isDeliveryIdle: () => true, }); const callback = vi.fn().mockReturnValue(EventConsumerResult.NotConsumed); @@ -435,6 +440,7 @@ describe('EventsConsumer', () => { const consumer = new EventsConsumer([], { onUnconsumedEvent, getPromiseQueue: () => Promise.resolve(), + isDeliveryIdle: () => true, }); const callback = vi.fn().mockReturnValue(EventConsumerResult.NotConsumed); @@ -455,6 +461,7 @@ describe('EventsConsumer', () => { const consumer = new EventsConsumer([event], { onUnconsumedEvent, getPromiseQueue: () => Promise.resolve(), + isDeliveryIdle: () => true, }); const callback1 = vi .fn() @@ -481,6 +488,174 @@ describe('EventsConsumer', () => { }); }); + describe('parking events that carry no ordering claim', () => { + /** + * A log event of a real type. The rest of this file uses a mock shape with + * no `eventType` at all, which is deliberately unparkable, so parking + * tests need events the consumer recognizes. + */ + function logEvent(eventType: Event['eventType'], id: string): Event { + // `eventId` as well as the mock shape's `id`: the consumer reports the + // former, the matcher below keys on the latter. + return createMockEvent({ id, eventId: id, eventType } as Partial); + } + + /** Consumes exactly the events whose id is in `ids`, once each. */ + function consumerFor(ids: string[]) { + const seen: string[] = []; + const callback = (event: Event | null) => { + if (event && ids.includes(event.id) && !seen.includes(event.id)) { + seen.push(event.id); + return EventConsumerResult.Consumed; + } + return EventConsumerResult.NotConsumed; + }; + return { seen, callback }; + } + + it('walks past an unclaimed hook_received instead of declaring divergence', async () => { + const hook = logEvent('hook_received', 'hook-1'); + const wait = logEvent('wait_created', 'wait-1'); + const onUnconsumedEvent = vi.fn(); + const consumer = new EventsConsumer([hook, wait], { + onUnconsumedEvent, + getPromiseQueue: () => Promise.resolve(), + isDeliveryIdle: () => true, + }); + const waits = consumerFor(['wait-1']); + + consumer.subscribe(waits.callback); + + // The hook belongs to a consumer this replay has not registered. The + // wait behind it is this replay's own decision and must still land. + await vi.waitFor(() => { + expect(waits.seen).toEqual(['wait-1']); + }); + expect(consumer.eventIndex).toBe(2); + expect(onUnconsumedEvent).not.toHaveBeenCalled(); + }); + + it('delivers a parked event to a consumer that subscribes later', async () => { + const hook = logEvent('hook_received', 'hook-1'); + const wait = logEvent('wait_created', 'wait-1'); + const onUnconsumedEvent = vi.fn(); + const consumer = new EventsConsumer([hook, wait], { + onUnconsumedEvent, + getPromiseQueue: () => Promise.resolve(), + isDeliveryIdle: () => true, + }); + const waits = consumerFor(['wait-1']); + consumer.subscribe(waits.callback); + await vi.waitFor(() => { + expect(waits.seen).toEqual(['wait-1']); + }); + + const hooks = consumerFor(['hook-1']); + consumer.subscribe(hooks.callback); + + await vi.waitFor(() => { + expect(hooks.seen).toEqual(['hook-1']); + }); + expect(onUnconsumedEvent).not.toHaveBeenCalled(); + }); + + it('replays a parked event under the index it held in the log', async () => { + const hook = logEvent('hook_received', 'hook-1'); + const wait = logEvent('wait_created', 'wait-1'); + const consumer = new EventsConsumer([hook, wait], { + onUnconsumedEvent: vi.fn(), + getPromiseQueue: () => Promise.resolve(), + isDeliveryIdle: () => true, + }); + consumer.subscribe(consumerFor(['wait-1']).callback); + await vi.waitFor(() => { + expect(consumer.eventIndex).toBe(2); + }); + + // Delivery barriers are registered under whatever `eventIndex` reads at + // consumption time, so a late delivery must still make the ordering + // claim its log position gave it β€” index 0, not the walk's 2. + let indexAtDelivery: number | undefined; + consumer.subscribe((event) => { + if (event?.id !== 'hook-1') { + return EventConsumerResult.NotConsumed; + } + indexAtDelivery = consumer.eventIndex; + return EventConsumerResult.Finished; + }); + + await vi.waitFor(() => { + expect(indexAtDelivery).toBe(0); + }); + // The walk pointer is restored, not left behind at the parked index. + expect(consumer.eventIndex).toBe(2); + }); + + it('still declares divergence for an unclaimed replay-origin event', async () => { + const step = logEvent('step_created', 'step-1'); + const unconsumedReceived = withResolvers(); + const consumer = new EventsConsumer([step], { + onUnconsumedEvent: unconsumedReceived.resolve, + getPromiseQueue: () => Promise.resolve(), + isDeliveryIdle: () => true, + }); + + consumer.subscribe(() => EventConsumerResult.NotConsumed); + + expect(await unconsumedReceived.promise).toEqual(step); + }); + + it('reports what it is still holding when the walk stops', async () => { + const hook = logEvent('hook_received', 'hook-1'); + const late = logEvent('hook_received', 'hook-2'); + const wait = logEvent('wait_created', 'wait-1'); + const consumer = new EventsConsumer([hook, late, wait], { + onUnconsumedEvent: vi.fn(), + getPromiseQueue: () => Promise.resolve(), + isDeliveryIdle: () => true, + }); + expect(consumer.parkedSummary).toBeUndefined(); + + consumer.subscribe(consumerFor(['wait-1']).callback); + await vi.waitFor(() => { + expect(consumer.eventIndex).toBe(3); + }); + + // Both hooks were walked past. A suspension is not a settling point, so + // the state goes on the span instead of failing the run: the oldest one + // held is what a query across a run's spans keys on. + expect(consumer.parkedSummary).toEqual({ + count: 2, + eventId: 'hook-1', + eventType: 'hook_received', + }); + + // Once a consumer claims them the run is holding nothing, and the + // attribute stops appearing on later spans. + consumer.subscribe(consumerFor(['hook-1', 'hook-2']).callback); + await vi.waitFor(() => { + expect(consumer.parkedSummary).toBeUndefined(); + }); + }); + + it('declares divergence for an event still parked once the run has ended', async () => { + const hook = logEvent('hook_received', 'hook-1'); + const completed = logEvent('run_completed', 'done-1'); + const unconsumedReceived = withResolvers(); + const consumer = new EventsConsumer([hook, completed], { + onUnconsumedEvent: unconsumedReceived.resolve, + getPromiseQueue: () => Promise.resolve(), + isDeliveryIdle: () => true, + }); + + // Nothing can subscribe for the hook after the run has finished, so + // parking it would silently drop it. + consumer.subscribe(consumerFor(['done-1']).callback); + + expect(await unconsumedReceived.promise).toEqual(hook); + }); + }); + describe('delivery-idle gate', () => { // An event nobody claims is only evidence of divergence once the workflow // VM has stopped reacting. While a delivery is in flight the walk is diff --git a/packages/core/src/events-consumer.ts b/packages/core/src/events-consumer.ts index 2367000eab..36647ed96c 100644 --- a/packages/core/src/events-consumer.ts +++ b/packages/core/src/events-consumer.ts @@ -9,23 +9,110 @@ import { eventsLogger } from './logger.js'; */ export const DEFERRED_CHECK_DELAY_MS = 100; +/** + * Floor for the deferred-check delay, so a too-low override can't manufacture + * spurious divergence (each false positive burns a divergence-recovery retry + * and can escalate to a terminal `CorruptedEventLogError`). + * + * Exported so tests needing the shortest legal delay can ask for it instead of + * hardcoding a number this floor would silently clamp up. + */ +export const MIN_DEFERRED_CHECK_DELAY_MS = 10; + /** * Effective deferred-check delay. Override: `WORKFLOW_DEFERRED_CHECK_DELAY_MS`. * * Unlike the other timing knobs this is not a polling interval but a * determinism safety margin: firing the unconsumed-event check before the * cross-VM subscribe() chain has landed rejects a healthy run with - * `ReplayDivergenceError`. Floored at 10ms so a too-low override can't - * manufacture spurious divergence (each false positive burns a - * divergence-recovery retry and can escalate to a terminal - * `CorruptedEventLogError`). + * `ReplayDivergenceError`. */ const getDeferredCheckDelayMs = (): number => envNumber('WORKFLOW_DEFERRED_CHECK_DELAY_MS', DEFERRED_CHECK_DELAY_MS, { integer: true, - min: 10, + min: MIN_DEFERRED_CHECK_DELAY_MS, }); +/** + * Event types the ordered walk may step over and deliver later. + * + * Membership is not about who wrote the event. It is about whether the event + * can reach the head of the walk with nothing registered to consume it, which + * is the only situation parking exists for. + * + * Most types cannot. They are replay-origin: a replay emits them, in the order + * its code reaches them, so their position in the log is the record of what + * that replay decided. Reaching one of those out of order means this replay + * decided differently than the log holds, which is divergence and nothing else. + * + * Step lifecycle events are not replay-origin, and are still absent, because + * they always have a claimant. `step()` subscribes its consumer before the + * step's first event can exist; `step_created` is ordered, so the walk cannot + * pass it unless a consumer takes it; and that consumer stays subscribed until + * `step_completed` or `step_failed`, after which the World refuses any further + * write for that step. There is no window in which a replay knows about a step + * and has nothing registered to consume its events, so parking them would only + * defer reports of what is divergence either way. The same holds for + * `wait_created` and its consumer. `wait_completed` is listed anyway: a sleep + * can also be completed out of band, by the API that force-completes pending + * waits, and tolerating a stray one is the cheaper direction to be wrong in. + * + * An allowlist rather than the complement of the ordered set, so a type this + * file has not been taught about keeps the strict old behaviour. + * + * `hook_disposed` is deliberately absent despite being about a hook: it is + * written when the workflow's own `using` scope exits, so it is replay-origin. + * + * `attr_set` is listed by type even though a given instance of it may be + * replay-origin, since it is replay-origin when its writer is the workflow. + * Splitting that out per event was considered and rejected. A replay that + * reaches one of its own writes out of position has diverged, but a replay that + * reaches an event it did NOT write, sitting where its own write would go, has + * not: the writer field says who wrote the event, and not whether this replay + * is the same one. Guessing wrong in that direction fails healthy runs, which + * is the failure this file exists to stop, so the whole type is tolerated. The + * cost is that a divergence involving `attr_set` surfaces at the end of the + * replay, through `strandedEvent`, rather than at the offending event. + */ +const PARKABLE_EVENT_TYPES: ReadonlySet = new Set([ + 'hook_received', + 'hook_conflict', + 'wait_completed', + 'attr_set', + 'run_cancelled', +]); + +/** + * Parkable types that can only ever resolve their correlation id once. + * + * A second one for the same id is not a delivery this replay has not reached + * yet: it is a resolution for something already resolved, which no consumer + * this replay or any later one registers can ever claim. `hook_received` is + * absent because a hook legitimately fires many times under one id. + * + * Must stay a subset of {@link PARKABLE_EVENT_TYPES}: {@link park} is the only + * reader of what this records, and it rejects a non-parkable type before it + * looks, so an entry outside that set is dead weight on a hot path. + */ +const ONE_SHOT_EVENT_TYPES: ReadonlySet = new Set([ + 'wait_completed', +]); + +/** Identifies the thing a one-shot resolution event resolves. */ +function resolutionKey(eventType: string, correlationId: string): string { + return `${eventType}:${correlationId}`; +} + +/** + * Types that end a run. Once one is in the log, no consumer will ever be + * registered again, so a parked event still parked here will never be claimed. + */ +const TERMINAL_EVENT_TYPES: ReadonlySet = new Set([ + 'run_completed', + 'run_failed', + 'run_cancelled', +]); + export enum EventConsumerResult { /** * Callback consumed the event, but should not be removed from the callbacks list @@ -71,16 +158,36 @@ export interface EventsConsumerOptions { * flight means the workflow VM is mid-reaction, and an event it has not * claimed yet is an event it has not reached yet. * - * Defaults to always-idle so the tests that drive a consumer with no - * orchestrator context keep the pre-existing timing. + * Required rather than defaulting to always-idle: always-idle is exactly the + * pre-gate behaviour, so a defaulted option would let a construction site opt + * a whole replay path back out without saying so. Tests that drive a consumer + * with no orchestrator context pass `() => true` to keep the pre-existing + * timing, and say so at the call site. */ - isDeliveryIdle?: () => boolean; + isDeliveryIdle: () => boolean; } export class EventsConsumer { eventIndex: number; readonly events: Event[]; readonly callbacks: EventConsumerCallback[] = []; + /** + * Events the ordered walk stepped over because nobody claimed them and their + * type carries no ordering claim. Each keeps the index it held in the log: + * consumers read {@link eventIndex} at consumption time to order their + * delivery against the rest of the log, and a late delivery must still make + * the claim its position gave it. + * + * Held in log order, drained in log order, and drained before every offer so + * a consumer registered after the walk passed the event still receives it. + */ + private readonly parked: { event: Event; index: number }[] = []; + /** + * Correlation ids of the {@link ONE_SHOT_EVENT_TYPES} events consumed so + * far, so a second resolution for one of them is recognized as unclaimable + * rather than parked for a consumer that cannot exist. + */ + private readonly resolved = new Set(); private onConsumedEvent?: (event: Event) => void; private onUnconsumedEvent: (event: Event) => void; private getPromiseQueue: () => Promise; @@ -98,7 +205,42 @@ export class EventsConsumer { this.onConsumedEvent = options.onConsumedEvent; this.onUnconsumedEvent = options.onUnconsumedEvent; this.getPromiseQueue = options.getPromiseQueue; - this.isDeliveryIdle = options.isDeliveryIdle ?? (() => true); + this.isDeliveryIdle = options.isDeliveryIdle; + } + + /** + * The oldest event the walk stepped over that no consumer has claimed yet, + * if any. Parking is a bet that a consumer will be registered later, so at + * any point where no consumer ever will be again β€” the replay finishing is + * the definitive one β€” this answers which event the bet lost on. + */ + get strandedEvent(): Event | undefined { + return this.parked[0]?.event; + } + + /** + * What the walk is still holding, or `undefined` when it holds nothing. + * + * Read at every point a replay stops, including the suspensions that are not + * settling points, so the held state reaches telemetry. A replay cannot tell + * a delivery awaiting a later consumer from one no consumer will ever + * register, so it reports rather than decides: the same `eventId` reported on + * suspension after suspension of one run is the shape that says the bet + * parking made is not going to pay off, and that shape is only visible across + * replays. + */ + get parkedSummary(): + | { count: number; eventId: string; eventType: Event['eventType'] } + | undefined { + const oldest = this.parked[0]?.event; + if (!oldest) { + return undefined; + } + return { + count: this.parked.length, + eventId: oldest.eventId, + eventType: oldest.eventType, + }; } append(events: Event[]): void { @@ -160,26 +302,42 @@ export class EventsConsumer { // case no callback consumes the event and we fall through to the // cross-VM-safe deferred unconsumed-event check below, exactly as before. while (true) { + // Before every offer, not just on subscribe: a callback registered by + // the work this same pass kicked off may be the owner of something + // parked, and the parked event's delivery is ordered ahead of the head + // event's by the index it holds. + this.drainParked(); const currentEvent = this.events[this.eventIndex] ?? null; - if (!this.consumeOne(currentEvent)) { - // No callback consumed the current event; handle the terminal case. - this.handleUnconsumed(currentEvent); + const consumed = this.offer(currentEvent); + if (consumed) { + this.eventIndex++; + } + if (currentEvent === null) { + // End of log. Consumers return NotConsumed for the `null` sentinel + // (the one that recognizes it as its own boundary schedules the + // suspension as a side effect and still declines it), so the drain + // stops here rather than spinning past the end. `consumed` is only + // true for a callback that claims the sentinel outright, which no + // production consumer does. + if (!consumed) { + this.handleEndOfLog(); + } return; } - // A real event was consumed β€” advance to the next in the same pass. A - // consumed `null` sentinel never returns true (see consumeOne), so the - // synchronous drain can't spin past the end of the log. + if (!consumed) { + this.scheduleUnconsumedCheck(currentEvent, true); + return; + } + // A real event was consumed β€” advance to the next in the same pass. } }; /** * Offer `currentEvent` to each registered callback in turn. Returns true - * when a callback consumed a real (non-null) event and the drain should - * advance to the next event in the same synchronous pass; false otherwise - * (nothing consumed it, or the consumed event was the end-of-events - * sentinel). + * when a callback consumed it. Does not move {@link eventIndex}: the ordered + * walk and the parked drain advance differently, so each does its own. */ - private consumeOne(currentEvent: Event | null): boolean { + private offer(currentEvent: Event | null): boolean { for (let i = 0; i < this.callbacks.length; i++) { const callback = this.callbacks[i]; let handled = EventConsumerResult.NotConsumed; @@ -195,63 +353,163 @@ export class EventsConsumer { continue; } if (currentEvent !== null) { + if ( + currentEvent.correlationId && + ONE_SHOT_EVENT_TYPES.has(currentEvent.eventType) + ) { + this.resolved.add( + resolutionKey(currentEvent.eventType, currentEvent.correlationId) + ); + } this.notifyConsumedEvent(currentEvent); } - // consumer handled this event, so increase the event index - this.eventIndex++; // remove the callback if it has finished if (handled === EventConsumerResult.Finished) { this.callbacks.splice(i, 1); } - // Continue draining only for real events. Real consumers return - // NotConsumed for the `null` sentinel, but guard against a pathological - // callback consuming it so the drain never spins past end-of-log. - return currentEvent !== null; + return true; } return false; } - private handleUnconsumed(currentEvent: Event | null) { + /** + * Offer everything parked, oldest first, until a pass claims nothing. + * + * Each offer runs with {@link eventIndex} moved back to the position the + * parked event held in the log, because that is the position its consumer + * will register a delivery barrier under. Restoring the walk pointer + * afterwards is what keeps the two pointers from interfering. + */ + private drainParked(): void { + let progressed = this.parked.length > 0; + while (progressed) { + progressed = false; + for (let i = 0; i < this.parked.length; i++) { + const entry = this.parked[i]; + const walkIndex = this.eventIndex; + this.eventIndex = entry.index; + let consumed: boolean; + try { + consumed = this.offer(entry.event); + } finally { + this.eventIndex = walkIndex; + } + if (consumed) { + this.parked.splice(i, 1); + // A `Finished` callback was spliced out of the list this pass, so + // restart rather than keep walking a mutated array. + progressed = this.parked.length > 0; + break; + } + } + } + } + + /** + * Step the ordered walk over an event nobody claimed, holding on to it for a + * later consumer. Returns false when the event's type makes its position a + * decision record, which is the one case where nobody claiming it means the + * replay diverged. + */ + private park(event: Event): boolean { + if (!PARKABLE_EVENT_TYPES.has(event.eventType)) { + return false; + } + if ( + event.correlationId && + this.resolved.has(resolutionKey(event.eventType, event.correlationId)) + ) { + return false; + } + this.parked.push({ event, index: this.eventIndex }); + this.eventIndex++; + eventsLogger.debug('Parked an unclaimed event for later delivery', { + eventId: event.eventId, + eventType: event.eventType, + correlationId: event.correlationId, + parked: this.parked.length, + }); + return true; + } + + private handleEndOfLog() { + // Everything still parked is waiting for a consumer some later replay will + // register, which is the whole point of parking β€” except once the log + // already holds the run's terminal event, because then there is no later + // replay and no consumer will ever come. + if (this.parked.length === 0) { + return; + } + const last = this.events.at(-1); + if (!last || !TERMINAL_EVENT_TYPES.has(last.eventType)) { + // A later replay is still expected, so nothing here is decidable and + // escalating would fail the healthy runs parking exists to keep alive. + // Reaching the end of the log holding something is not by itself a fault, + // so this stays at `debug`; {@link parkedSummary} is what carries the + // state to the span, where a run that keeps stopping on the same held + // event is visible and a single replay's view is not. + eventsLogger.debug('Reached the end of the log still holding events', { + eventId: this.parked[0].event.eventId, + eventType: this.parked[0].event.eventType, + correlationId: this.parked[0].event.correlationId, + parked: this.parked.length, + }); + return; + } + this.scheduleUnconsumedCheck(this.parked[0].event, false); + } + + private scheduleUnconsumedCheck(currentEvent: Event, mayPark: boolean) { // All callbacks returned NotConsumed for the current event. - // If the current event is non-null (a real event, not end-of-events), - // schedule a deferred check. We chain onto the promiseQueue so that any + // Schedule a deferred check. We chain onto the promiseQueue so that any // pending async work (e.g., deserialization/decryption that triggers // resolve() β†’ user code β†’ subscribe()) completes first. If the event - // is still unconsumed after the queue drains, it's truly orphaned. - if (currentEvent !== null) { - const checkVersion = ++this.unconsumedCheckVersion; - this.pendingUnconsumedCheck = this.getPromiseQueue() - .then( - // Yield once after the first queue drain so promise chains resumed by - // that drain can run across the VM boundary and append any follow-up - // async work (for example: step_completed resolves -> for-await loop - // resumes -> the next hook payload starts hydrating). - () => new Promise((resolve) => setTimeout(resolve, 0)) - ) - .then(() => this.getPromiseQueue()) - .then(() => { - // Wait out any delivery still in flight before starting the timer. - // The queue draining says the host has no hydration work left; it - // does not say the VM has finished reacting to what was hydrated. - this.whenDeliveryIdle(checkVersion, () => { - // Use a delayed setTimeout once deliveries are idle. The delay must - // be long enough for promise chains to propagate across the VM - // boundary (from resolve() in the host context through to the - // workflow code calling subscribe() in the VM context). Node.js - // does not guarantee that setTimeout(0) fires after all - // cross-context microtasks settle, so we use a small but non-zero - // delay. Any subscribe() call that arrives during this window will - // cancel the check via version invalidation + clearTimeout. - this.pendingUnconsumedTimeout = setTimeout(() => { - this.pendingUnconsumedTimeout = null; - if (this.unconsumedCheckVersion === checkVersion) { - this.pendingUnconsumedCheck = null; - this.onUnconsumedEvent(currentEvent); + // is still unconsumed after the queue drains, it's truly orphaned β€” or, + // when its type carries no ordering claim, parked for a later consumer. + const checkVersion = ++this.unconsumedCheckVersion; + this.pendingUnconsumedCheck = this.getPromiseQueue() + .then( + // Yield once after the first queue drain so promise chains resumed by + // that drain can run across the VM boundary and append any follow-up + // async work (for example: step_completed resolves -> for-await loop + // resumes -> the next hook payload starts hydrating). + () => new Promise((resolve) => setTimeout(resolve, 0)) + ) + .then(() => this.getPromiseQueue()) + .then(() => { + // Wait out any delivery still in flight before starting the timer. + // The queue draining says the host has no hydration work left; it + // does not say the VM has finished reacting to what was hydrated. + this.whenDeliveryIdle(checkVersion, () => { + // Use a delayed setTimeout once deliveries are idle. The delay must + // be long enough for promise chains to propagate across the VM + // boundary (from resolve() in the host context through to the + // workflow code calling subscribe() in the VM context). Node.js does + // not guarantee that setTimeout(0) fires after all cross-context + // microtasks settle, so we use a small but non-zero delay. Any + // subscribe() call that arrives during this window will cancel the + // check via version invalidation + clearTimeout. + this.pendingUnconsumedTimeout = setTimeout(() => { + this.pendingUnconsumedTimeout = null; + if (this.unconsumedCheckVersion !== checkVersion) { + return; + } + this.pendingUnconsumedCheck = null; + if (mayPark) { + if (this.events[this.eventIndex] !== currentEvent) { + // An append() drain claimed it while the check was in flight. + // Only subscribe() cancels the check, so this is reachable. + return; + } + if (this.park(currentEvent)) { + this.consume(); + return; } - }, getDeferredCheckDelayMs()); - }); + } + this.onUnconsumedEvent(currentEvent); + }, getDeferredCheckDelayMs()); }); - } + }); } /** @@ -267,14 +525,34 @@ export class EventsConsumer { * batch of N parallel step results leaves N-1 of them on that detached path * with the queue already drained, so the walk sits on the ordered event the * VM is about to draw and the window is the only thing standing between a - * healthy run and `ReplayDivergenceError`. On a backend whose deliveries take - * longer than the window, that bet loses: the local race repro corrupts 34 of - * 42 runs at a 10ms window and 0 of 114 at 100ms, on the same event logs. + * healthy run and `ReplayDivergenceError`. + * + * Shortening the window shows that mechanism directly: on identical event logs + * the local race repro corrupts 34 of 42 runs at a 10ms window and 0 of 114 at + * the 100ms default. That measures how the bet loses, not that the default + * loses it, and no measurement of a delivery outrunning 100ms exists either + * way. So read this as retiring the bet rather than as repairing an observed + * failure of that number: the delay is a user-settable env override, which + * leaves the old behaviour one configuration away from losing on any backend. * * Termination is `hasParkedCommittedDelivery`'s: it counts only deliveries * that resolve on their own, so nothing here can gate its own retirement. A * genuinely orphaned event has no delivery to wait on and reaches `fn` on the * first poll. + * + * What the gate gives up: for the ordered events that still reach + * `onUnconsumedEvent` rather than {@link park}, this stops being the thing + * that catches a diverged log while a delivery is in flight. The suspension + * and this check now wake from the same `isDeliveryIdle` edge, and + * `scheduleWhenIdle` fires on the first timer tick after idle while this waits + * a further `getDeferredCheckDelayMs()`. So a run with a pending `sleep()` + * suspends first, and `onWorkflowError` drops the divergence arriving second + * (its `'suspended'` branch demotes to `'replay'` and surfaces nothing), + * leaving a later `resume()` to decline into a cold replay. Pre-gate the + * suspension already won that race whenever the delivery landed inside the + * fixed window, so what changed is that the outcome stopped depending on + * timing. Nothing should treat this check as the mechanism that reports + * divergence on a log the run is still delivering into. */ private whenDeliveryIdle(checkVersion: number, fn: () => void): void { const poll = () => { @@ -291,7 +569,13 @@ export class EventsConsumer { return; } // Held in the same field the fired check uses so subscribe() cancels a - // poll in progress exactly as it cancels the check itself. + // poll in progress too. The two cancellations are not identical: for the + // poll it is the version bump that does the work and the clearTimeout is + // belt-and-braces. Two state machines write this one field, so a poll + // invalidated between scheduling and firing can null out a handle the + // live chain has since stored, which is why every path out of `poll` and + // out of the fired check re-checks the version rather than trusting the + // handle. this.pendingUnconsumedTimeout = setTimeout(poll, 0); }); }; diff --git a/packages/core/src/hook-sleep-interaction.test.ts b/packages/core/src/hook-sleep-interaction.test.ts index 754ee1a138..9505cbb17c 100644 --- a/packages/core/src/hook-sleep-interaction.test.ts +++ b/packages/core/src/hook-sleep-interaction.test.ts @@ -4,7 +4,6 @@ import type { Event } from '@workflow/world'; import * as nanoid from 'nanoid'; import { monotonicFactory } from 'ulid'; import { afterEach, describe, expect, it, vi } from 'vitest'; -import { createCorrelationIdGenerator } from './correlation-id.js'; import { EventsConsumer } from './events-consumer.js'; import { WorkflowSuspension } from './global.js'; import type { WorkflowOrchestratorContext } from './private.js'; @@ -49,6 +48,8 @@ function setupWorkflowContext(events: Event[]): WorkflowOrchestratorContext { replayPayloadCache: new ReplayPayloadCache(undefined), globalThis: context.globalThis, eventsConsumer: new EventsConsumer(events, { + // Fake context: no deliveries are modeled, so the gate is a no-op here. + isDeliveryIdle: () => true, onUnconsumedEvent: (event) => { ctxRef.current?.onWorkflowError( new WorkflowRuntimeError( @@ -59,14 +60,7 @@ function setupWorkflowContext(events: Event[]): WorkflowOrchestratorContext { getPromiseQueue: () => promiseQueueHolder.current, }), invocationsQueue: new Map(), - generateCorrelationId: createCorrelationIdGenerator({ - seed: 'test', - fixedTimestamp: workflowStartedAt, - positional: () => ulid(workflowStartedAt), - // The event logs in this file hardcode correlation ids the run-wide - // shared sequence minted, so replay only matches under that scheme. - perKind: false, - }), + generateUlid: () => ulid(workflowStartedAt), generateNanoid: nanoid.customRandom(nanoid.urlAlphabet, 21, (size) => new Uint8Array(size).map(() => 256 * context.globalThis.Math.random()) ), diff --git a/packages/core/src/private.ts b/packages/core/src/private.ts index f7bedcd471..929fff699e 100644 --- a/packages/core/src/private.ts +++ b/packages/core/src/private.ts @@ -4,7 +4,6 @@ import { withResolvers } from '@workflow/utils'; import type { WorldCapabilities } from '@workflow/world'; -import type { CorrelationIdGenerator } from './correlation-id.js'; import type { EventsConsumer } from './events-consumer.js'; import type { QueueItem } from './global.js'; import type { ReplayPayloadCache } from './replay-payload-cache.js'; @@ -154,11 +153,11 @@ export interface WorkflowOrchestratorContext { invocationsQueue: Map; onWorkflowError: (error: Error) => void; /** - * Mints a correlation id body for one entity family. Every entity a replay - * creates draws from here, and the family is what keeps a disagreement about - * one family's count from renumbering another's. + * Mints the ULID body of a correlation id. Every entity a replay creates + * draws from this one monotonic sequence, so an id is an ordinal over the + * whole run and both replays of a run must draw in the same order. */ - generateCorrelationId: CorrelationIdGenerator; + generateUlid: () => string; generateNanoid: () => string; /** * Sequential promise queue that ensures all event-driven promise resolutions diff --git a/packages/core/src/runtime.test.ts b/packages/core/src/runtime.test.ts index fdaa4ccb39..76724c199a 100644 --- a/packages/core/src/runtime.test.ts +++ b/packages/core/src/runtime.test.ts @@ -947,17 +947,83 @@ describe('workflowEntrypoint replay guards', () => { deploymentId: 'test-deployment', }; + // `hook_created` records a hook a replay decided to create, so its + // position and identity are that replay's decision record: one the current + // replay does not create is divergence on the spot. A `hook_received` here + // would not be, since a delivery nobody claims is parked for a later + // consumer (see 'suspends rather than failing on a hook delivery that + // matches no hook' below). const events: Event[] = [ { eventId: 'event-0', runId: workflowRun.runId, - eventType: 'hook_received', + eventType: 'hook_created', correlationId: 'hook_01HK153X00VFKAJV9XFN9JXXRS', eventData: { token: 'wrong-token', + isWebhook: false, + }, + createdAt: new Date('2024-01-01T00:00:00.000Z'), + }, + ]; + + const createdEvents: unknown[] = []; + const queueCalls: QueueCall[] = []; + await runWorkflowHandlerWithEvents( + `const createHook = globalThis[Symbol.for("WORKFLOW_CREATE_HOOK")]; + async function workflow() { + const hook = createHook({ token: 'expected-token' }); + const payload = await hook; + return payload.message; + }${getWorkflowTransformCode('workflow')}`, + workflowRun, + events, + { createdEvents, queueCalls } + ); + + expect(createdEvents).not.toContainEqual( + expect.objectContaining({ eventType: 'run_failed' }) + ); + expect(queueCalls.map((c) => c.message)).toContainEqual( + expect.objectContaining({ + replayDivergence: { eventId: 'event-0', count: 1 }, + }) + ); + }); + + it('suspends rather than failing on a hook delivery that matches no hook', async () => { + const ops: Promise[] = []; + const workflowRun: WorkflowRun = { + runId: 'wrun_runtime_hook_parked', + workflowName: 'workflow', + status: 'running', + input: await dehydrateWorkflowArguments( + [], + 'wrun_runtime_hook_parked', + undefined, + ops + ), + createdAt: new Date('2024-01-01T00:00:00.000Z'), + updatedAt: new Date('2024-01-01T00:00:00.000Z'), + startedAt: new Date('2024-01-01T00:00:00.000Z'), + deploymentId: 'test-deployment', + }; + + // A delivery for a hook this replay never registers a consumer for. A + // writer that raced this replay can leave one in the log legitimately, so + // it is held for a consumer a later replay may register instead of ending + // the run. + const events: Event[] = [ + { + eventId: 'event-0', + runId: workflowRun.runId, + eventType: 'hook_received', + correlationId: 'hook_01HK153X00VFKAJV9XFN9JXXRS', + eventData: { + token: 'some-other-hook', payload: await dehydrateStepReturnValue( { message: 'hello' }, - 'wrun_runtime_hook_guard', + 'wrun_runtime_hook_parked', undefined, ops ), @@ -983,11 +1049,12 @@ describe('workflowEntrypoint replay guards', () => { expect(createdEvents).not.toContainEqual( expect.objectContaining({ eventType: 'run_failed' }) ); - expect(queueCalls.map((c) => c.message)).toContainEqual( - expect.objectContaining({ - replayDivergence: { eventId: 'event-0', count: 1 }, - }) + expect(createdEvents).toContainEqual( + expect.objectContaining({ eventType: 'hook_created' }) ); + expect( + queueCalls.filter((call) => 'replayDivergence' in (call.message ?? {})) + ).toEqual([]); }); it('replays attribute events before executing a step that loses the same race', async () => { diff --git a/packages/core/src/runtime.ts b/packages/core/src/runtime.ts index 29fd659ca3..5b34574433 100644 --- a/packages/core/src/runtime.ts +++ b/packages/core/src/runtime.ts @@ -73,13 +73,16 @@ import { handleHealthCheckMessage, insertEventByEventId, isPreconditionGuardEnabled, + isSlotGapCheckEnabled, type LoadedEventLog, loadWorkflowRunEvents, memoizeEncryptionKey, + mergeReportedEvents, parseHealthCheckPayload, preconditionEventDelta, preconditionSnapshotParams, queueMessage, + settleEventSlotGap, withHealthCheck, } from './runtime/helpers.js'; import { @@ -2499,13 +2502,34 @@ export function workflowEntrypoint( for (const waitEvent of waitsToComplete) { try { - await createEvent(waitEvent, { + const created = await createEvent(waitEvent, { requestId, ...preconditionSnapshotParams( eventLog.events, eventLog.cursor ), }); + // Bump-and-report: fold what this write skipped over + // into the snapshot the remaining waits are guarded + // against, so each asks for a slot above it. + // + // Only a complete answer. `hasMore` means the World + // returned part of what it was asked for, and the + // missing-completion check below is what decides + // whether this handler still has to fetch. Folding in + // a partial page would make the log look like it + // holds the completion when the rest of the page is + // still unread, so the fetch would be skipped on a + // snapshot that is short of the World's. + if ( + created.events?.length && + created.hasMore !== true + ) { + mergeReportedEvents( + eventLog.events, + created.events + ); + } } catch (err) { if (EntityConflictError.is(err)) { runtimeLogger.info( @@ -2586,6 +2610,30 @@ export function workflowEntrypoint( } } + // A replay reads the log as the complete record of what + // has happened, so a position nothing occupies is + // indistinguishable from an event that never occurred and + // the branch it would have decided gets decided the other + // way. Failing here is the difference between a run that + // reports its own corruption and one that silently + // returns the wrong answer. + // + // A hole that is merely a write mid-commit fills in on + // its own, so settleEventSlotGap re-reads before + // concluding, and adopts whichever log it settled on. + if (isSlotGapCheckEnabled()) { + const settled = await settleEventSlotGap(runId, { + events: eventLog.events, + cursor: eventLog.cursor, + }); + eventLog = { ...settled.log, type: 'ready' }; + if (settled.gap !== undefined) { + throw new CorruptedEventLogError( + `Event log for run ${runId} has a hole at slot ${settled.gap.firstMissingSlot}: ${settled.gap.missingCount} of the ${settled.gap.maxSlot} slots up to the log's maximum hold no event.` + ); + } + } + // Completing elapsed waits refreshes the event snapshot. // A concurrent handler may have written the terminal run // event after the initial snapshot but before this @@ -2898,6 +2946,16 @@ export function workflowEntrypoint( }); return; } + if (suspensionResult.reportedEventCount > 0) { + // Bump-and-report merged events BELOW the tail and + // re-sorted the array to slot order, shifting every + // position the prewarm scan had already recorded. + // The cursor is deliberately left alone: the report + // is a lower bound on what was skipped, so the next + // incremental read still has to cover the same range. + replayPayloadCache.resetScan(); + } + // Open hooks/waits in the log as loaded for this // replay. This suspension's own hook/wait writes are // NOT in it β€” they never reach retention anyway, diff --git a/packages/core/src/runtime/helpers.test.ts b/packages/core/src/runtime/helpers.test.ts index c2e098f057..4623f006a4 100644 --- a/packages/core/src/runtime/helpers.test.ts +++ b/packages/core/src/runtime/helpers.test.ts @@ -1,5 +1,6 @@ import { PreconditionFailedError, WorkflowWorldError } from '@workflow/errors'; import type { Event, World } from '@workflow/world'; +import { slotToEventId } from '@workflow/world'; import { ulid } from 'ulid'; import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest'; import { bytesToBase64, deriveRunKeyPair, seal } from '../sealed-box.js'; @@ -12,15 +13,20 @@ import { } from '../serialization.js'; import { appendUniqueEvents, + findEventSlotGap, getWorkflowQueueName, handleHealthCheckMessage, healthCheck, insertEventByEventId, latestEventStateUpdatedAt, loadWorkflowRunEvents, + maxEventSlot, memoizeEncryptionKey, + mergeReportedEvents, preconditionEventDelta, preconditionSnapshotParams, + SLOT_GAP_RECHECK_ATTEMPTS, + settleEventSlotGap, } from './helpers.js'; // Mock the logger to suppress output during tests @@ -700,6 +706,256 @@ describe('preconditionSnapshotParams', () => { }); }); +describe('preconditionSnapshotParams on a slot-numbered run', () => { + let originalGuard: string | undefined; + + beforeEach(() => { + originalGuard = process.env.WORKFLOW_PRECONDITION_GUARD; + process.env.WORKFLOW_PRECONDITION_GUARD = '1'; + }); + + afterEach(() => { + if (originalGuard !== undefined) { + process.env.WORKFLOW_PRECONDITION_GUARD = originalGuard; + } else { + delete process.env.WORKFLOW_PRECONDITION_GUARD; + } + }); + + it('sends eventCount instead of the ULID triple', () => { + const events = [1, 2, 3].map((slot) => makeEvent(slotToEventId(slot))); + + expect(preconditionSnapshotParams(events, 'eid:abc')).toEqual({ + eventCount: 3, + }); + }); + + it('reports the highest slot, not the number of events', () => { + // A slot is claimed by the write that occupies it, and a write that then + // fails leaves it empty forever. Sending the count would make every later + // write in this run ask below the hole and be handed the same events back + // on every single create. + const events = [1, 2, 5].map((slot) => makeEvent(slotToEventId(slot))); + + expect(preconditionSnapshotParams(events, 'eid:abc')).toEqual({ + eventCount: 5, + }); + }); + + it('is invariant under the order the World returned the log in', () => { + const forward = [1, 2, 3].map((slot) => makeEvent(slotToEventId(slot))); + + expect(preconditionSnapshotParams([...forward].reverse(), null)).toEqual( + preconditionSnapshotParams(forward, null) + ); + }); + + it('omits eventCount when the guard is disabled', () => { + process.env.WORKFLOW_PRECONDITION_GUARD = '0'; + + expect( + preconditionSnapshotParams([makeEvent(slotToEventId(1))], null) + ).toEqual({}); + }); + + it('falls back to the ULID triple when one event is not a slot', () => { + // A log may not mix the two schemes. If it somehow does, the slot reading + // is meaningless, so the run is treated as ULID-numbered. + const time = 1_700_000_000_000; + const events = [makeEvent(slotToEventId(1)), makeUlidEvent(time)]; + + expect(preconditionSnapshotParams(events, null)).toEqual({ + stateUpdatedAt: time, + stateEventCount: 2, + }); + }); +}); + +describe('maxEventSlot', () => { + it('is undefined for a log with no slot ids', () => { + expect(maxEventSlot([])).toBeUndefined(); + expect(maxEventSlot([makeUlidEvent(1_700_000_000_000)])).toBeUndefined(); + }); +}); + +/** + * The hole check a replay runs over its loaded log. It gates whether the run + * executes at all, so it is one-sided in the opposite direction from the + * World's density counter: it reports a hole only where the log proves one, and + * says nothing about a log it cannot read as slots. + */ +describe('findEventSlotGap', () => { + const slotLog = (...slots: number[]) => + slots.map((slot) => makeEvent(slotToEventId(slot))); + + it('finds no hole in a dense log', () => { + expect(findEventSlotGap(slotLog(1, 2, 3))).toBeUndefined(); + }); + + it('names the hole and how much of the log is missing', () => { + expect(findEventSlotGap(slotLog(1, 2, 5))).toEqual({ + firstMissingSlot: 3, + missingCount: 2, + maxSlot: 5, + }); + }); + + it('reports the lowest hole when there is more than one', () => { + expect(findEventSlotGap(slotLog(1, 3, 5))).toEqual({ + firstMissingSlot: 2, + missingCount: 2, + maxSlot: 5, + }); + }); + + it('does not depend on the log being in slot order', () => { + // The loaded log is listed pages plus whatever a bump-and-report write + // handed back. mergeReportedEvents restores order, but a check that fails + // a run outright must not be the thing that notices when it did not. + expect(findEventSlotGap(slotLog(3, 1, 2))).toBeUndefined(); + expect(findEventSlotGap(slotLog(4, 1, 2))?.firstMissingSlot).toBe(3); + }); + + it('excuses a log missing only its reserved first slot', () => { + // `start()` posts run_created concurrently with the queue send, so a log + // read in that window legitimately begins at the second slot. + expect(findEventSlotGap(slotLog(2, 3))).toBeUndefined(); + }); + + it('still reports a hole above an absent first slot', () => { + expect(findEventSlotGap(slotLog(2, 4))).toEqual({ + firstMissingSlot: 3, + missingCount: 1, + maxSlot: 4, + }); + }); + + it('says nothing about a log it cannot read as slots', () => { + expect(findEventSlotGap([])).toBeUndefined(); + expect( + findEventSlotGap([makeUlidEvent(1_700_000_000_000)]) + ).toBeUndefined(); + // A ULID anywhere disarms it: the run is not slot-numbered, and a mixed + // log has no density to measure. + expect( + findEventSlotGap([ + ...slotLog(1, 2), + makeUlidEvent(1_700_000_000_000), + ...slotLog(9), + ]) + ).toBeUndefined(); + }); +}); + +/** + * The re-read that stands between a hole and a failed run. A hole can be one + * commit wide: the World allocates a slot inside the insert that occupies it, + * so a writer can commit a higher slot while a lower one is still in flight. + * Only a hole that survives the re-reads is a position no write will ever take. + */ +describe('settleEventSlotGap', () => { + beforeEach(() => { + eventsListMock.mockReset(); + }); + + const slotLog = (...slots: number[]) => + slots.map((slot) => makeEvent(slotToEventId(slot))); + + it('reports no gap for a log that is already dense', async () => { + const settled = await settleEventSlotGap('wrun_test', { + events: slotLog(1, 2, 3), + cursor: 'eid:c', + }); + + expect(settled.gap).toBeUndefined(); + // Nothing to settle, so nothing is re-read. + expect(eventsListMock).not.toHaveBeenCalled(); + }); + + it('adopts the log it re-read once the hole has filled in', async () => { + eventsListMock.mockResolvedValueOnce({ + data: slotLog(1, 2, 3), + cursor: 'eid:filled', + hasMore: false, + }); + + const settled = await settleEventSlotGap('wrun_test', { + events: slotLog(1, 3), + cursor: 'eid:stale', + }); + + expect(settled.gap).toBeUndefined(); + // The caller replays what settled, not the snapshot that looked holey. + expect(settled.log.events.map((e) => e.eventId)).toEqual( + slotLog(1, 2, 3).map((e) => e.eventId) + ); + expect(settled.log.cursor).toBe('eid:filled'); + expect(eventsListMock).toHaveBeenCalledTimes(1); + }); + + it('reports a hole that survives every re-read', async () => { + eventsListMock.mockResolvedValue({ + data: slotLog(1, 4), + cursor: 'eid:stuck', + hasMore: false, + }); + + const settled = await settleEventSlotGap('wrun_test', { + events: slotLog(1, 4), + cursor: 'eid:stuck', + }); + + expect(settled.gap).toEqual({ + firstMissingSlot: 2, + missingCount: 2, + maxSlot: 4, + }); + expect(eventsListMock).toHaveBeenCalledTimes(SLOT_GAP_RECHECK_ATTEMPTS); + }); +}); + +describe('mergeReportedEvents', () => { + it('restores slot order after folding in events below the tail', () => { + // Bump-and-report hands back events the writer had not seen, and they sit + // BELOW the write that reported them. Appending would leave the log in an + // order no replay can walk. + const target = [1, 4].map((slot) => makeEvent(slotToEventId(slot))); + + const added = mergeReportedEvents( + target, + [3, 2].map((slot) => makeEvent(slotToEventId(slot))) + ); + + expect(added).toBe(2); + expect(target.map((e) => e.eventId)).toEqual( + [1, 2, 3, 4].map(slotToEventId) + ); + }); + + it('is a no-op when every reported event is already present', () => { + const target = [1, 2].map((slot) => makeEvent(slotToEventId(slot))); + + expect(mergeReportedEvents(target, [makeEvent(slotToEventId(2))])).toBe(0); + expect(target).toHaveLength(2); + }); + + it('leaves a ULID log in receipt order', () => { + // Only a slot log has an id order the runtime may impose. A World that + // orders by (createdAt, eventId) would be reordered into a log it never + // produced. + const first = makeUlidEvent(1_700_000_000_000); + const second = makeUlidEvent(1_600_000_000_000); + const target = [first]; + + mergeReportedEvents(target, [second]); + + expect(target.map((e) => e.eventId)).toEqual([ + first.eventId, + second.eventId, + ]); + }); +}); + describe('appendUniqueEvents', () => { it('appends in receipt order', () => { const first = makeUlidEvent(1_700_000_000_000); diff --git a/packages/core/src/runtime/helpers.ts b/packages/core/src/runtime/helpers.ts index 862d907dd5..c2483d6285 100644 --- a/packages/core/src/runtime/helpers.ts +++ b/packages/core/src/runtime/helpers.ts @@ -4,13 +4,18 @@ import { WorkflowWorldError, } from '@workflow/errors'; import type { + CreateEventParams, + CreateEventRequest, Event, + EventResult, HealthCheckPayload, ValidQueueName, WorkflowRun, World, } from '@workflow/world'; import { + eventIdToSlot, + FIRST_EVENT_SLOT, getQueueTopicPrefix, HealthCheckPayloadSchema, HOOK_RESUME_INPUT_VERSION, @@ -717,6 +722,23 @@ export function isPreconditionGuardEnabled(): boolean { return process.env.WORKFLOW_PRECONDITION_GUARD !== '0'; } +/** + * Whether a replay refuses to run over a log with a hole in it (see + * {@link findEventSlotGap}). **On by default**; set + * `WORKFLOW_SLOT_GAP_CHECK=0` to replay across holes instead. + * + * The switch exists because the check trades one failure for another. A hole is + * a position claimed by a write that then failed, so most of them stand for an + * event that never happened and replaying past one is correct. But a hole + * standing for an event that *did* happen is indistinguishable from that, and + * replaying past that one produces a run whose result is wrong with nothing to + * show for it. Failing loudly is the recoverable side of the trade, and this is + * the way back out if a fleet turns out to carry benign holes. + */ +export function isSlotGapCheckEnabled(): boolean { + return process.env.WORKFLOW_SLOT_GAP_CHECK !== '0'; +} + /** * The `stateUpdatedAt` value to send with a replay-context event creation: the * *maximum* ULID time (epoch ms) over the events the runtime has loaded. Returns @@ -769,19 +791,202 @@ export function latestEventStateUpdatedAt(events: Event[]): number | undefined { return time; } +/** + * Merge the events a bump-and-report write handed back into the log it was + * derived from, and answer how many of them were new. + * + * Unlike {@link appendUniqueEvents}, this re-sorts. The reported events occupy + * slots *below* the write that reported them, so appending them would put them + * after events they precede β€” and on a slot-numbered run the id order is the + * World's canonical order, so restoring it is well defined rather than a guess. + * A run that is not slot-numbered cannot produce this report in the first + * place; the sort is skipped rather than applied to ids it cannot order. + */ +export function mergeReportedEvents( + target: Event[], + events: readonly Event[] +): number { + const before = target.length; + appendUniqueEvents(target, events); + const added = target.length - before; + if (added > 0 && maxEventSlot(target) !== undefined) { + target.sort((a, b) => + a.eventId < b.eventId ? -1 : a.eventId > b.eventId ? 1 : 0 + ); + } + return added; +} + +/** + * The highest slot the loaded log occupies, or `undefined` when the run is not + * slot-numbered. A run keeps the id scheme it was created under, so one event + * settles it for the whole log. + * + * The maximum, not the count, and the two are not interchangeable even though + * a healthy log makes them equal. A World hands a position to the insert that + * occupies it, so a write that never lands leaves no hole behind and the log + * stays dense. What the count cannot survive is a *partial* read: a log + * assembled from a truncated report, or read while a concurrent write is + * committing, holds fewer events than its highest position. Counting those + * would make the next write claim to have seen less than it has, so the World + * would report the same events back to it on every attempt. + * + * A hole below the maximum is therefore a property of the read, not of the log, + * which is what lets {@link settleEventSlotGap} re-read instead of giving up. + */ +export function maxEventSlot(events: Event[]): number | undefined { + let max: number | undefined; + for (const event of events) { + const slot = eventIdToSlot(event.eventId); + if (slot === null) { + return undefined; + } + if (max === undefined || slot > max) { + max = slot; + } + } + return max; +} + +/** A position the log skips over, described well enough to name in an error. */ +export interface EventSlotGap { + /** The lowest slot below the log's maximum that no event occupies. */ + firstMissingSlot: number; + /** How many slots below the maximum no event occupies. */ + missingCount: number; + /** The highest slot the log occupies. */ + maxSlot: number; +} + +/** + * The hole in a loaded log, or `undefined` when there is none to find. + * + * On a slot-numbered run the World allocates every position, so a log that + * holds `n` events below slot `n` is missing one. That matters before a replay + * and nowhere else: the replay reads the log as the complete record of what has + * happened, and an absent position is indistinguishable from an event that + * never occurred. The branch it would have decided gets decided the other way, + * and the run diverges quietly rather than failing. + * + * Order-independent, unlike the equivalent audit the World runs over a page it + * just read. A loaded log is assembled from listed pages plus whatever a + * bump-and-report write handed back, and while {@link mergeReportedEvents} + * restores id order, a check that can fail a healthy run should not depend on + * that having happened. + * + * The first slot is never counted. It belongs to `run_created`, which `start()` + * posts concurrently with the queue send, so a log read in that window + * legitimately begins at the second slot and fills in on its own. Every replay + * that races a run's own start would otherwise report a hole. + * + * Returns `undefined` for a log this cannot read as slots at all: an empty one, + * or a run numbered by ULID, where positions carry no density to check. + */ +export function findEventSlotGap( + events: readonly Event[] +): EventSlotGap | undefined { + const occupied = new Set(); + let maxSlot = 0; + for (const event of events) { + const slot = eventIdToSlot(event.eventId); + if (slot === null) { + return undefined; + } + occupied.add(slot); + if (slot > maxSlot) { + maxSlot = slot; + } + } + if (maxSlot === 0) { + return undefined; + } + const floor = occupied.has(FIRST_EVENT_SLOT) + ? FIRST_EVENT_SLOT + : FIRST_EVENT_SLOT + 1; + // Every slot is at or above `floor` by construction, so the log is dense + // exactly when it holds one event per position in `[floor, maxSlot]`. The + // scan below only runs once that has already answered no. + if (occupied.size === maxSlot - floor + 1) { + return undefined; + } + let firstMissingSlot: number | undefined; + let missingCount = 0; + for (let slot = floor; slot <= maxSlot; slot++) { + if (!occupied.has(slot)) { + firstMissingSlot ??= slot; + missingCount++; + } + } + if (firstMissingSlot === undefined) { + return undefined; + } + return { firstMissingSlot, missingCount, maxSlot }; +} + +/** + * How many times a detected hole is re-read before the log is taken at its + * word, and the backoff before each re-read (doubling per attempt). + * + * A hole can be transient. The World allocates a slot inside the insert that + * occupies it, so two concurrent writers can collide, one retry past the other, + * and the higher slot commit first β€” leaving a window in which the lower one is + * genuinely absent from a strongly-consistent read and fills in a moment later. + * The window is one commit wide, so a short backoff clears it; anything that + * survives all three re-reads is a position no write will ever occupy. + */ +export const SLOT_GAP_RECHECK_ATTEMPTS = 3; +const SLOT_GAP_RECHECK_BASE_DELAY_MS = 25; + +/** + * Re-read a log that looks holey until the hole fills in or the re-reads run + * out, and return the settled log alongside the hole that survived. + * + * Reads are strongly consistent, so a hole is not an artifact of *when* the log + * was read β€” but it can be an artifact of a write that had not committed yet + * (see {@link SLOT_GAP_RECHECK_ATTEMPTS}). Distinguishing the two costs a + * re-read, which is only ever paid by a replay that already found a hole. + * + * The reload is full rather than incremental: the missing position is below the + * log's maximum, so a cursor-anchored read starts past it and can never see it + * arrive. + */ +export async function settleEventSlotGap( + runId: string, + loaded: LoadedEventLog +): Promise<{ log: LoadedEventLog; gap: EventSlotGap | undefined }> { + let log = loaded; + let gap = findEventSlotGap(log.events); + for ( + let attempt = 0; + gap !== undefined && attempt < SLOT_GAP_RECHECK_ATTEMPTS; + attempt++ + ) { + await new Promise((resolve) => + setTimeout(resolve, SLOT_GAP_RECHECK_BASE_DELAY_MS * 2 ** attempt) + ); + log = await loadWorkflowRunEvents(runId); + gap = findEventSlotGap(log.events); + } + return { log, gap }; +} + /** * The precondition snapshot a replay-context event creation sends, describing * the event log the replay derived the event from. * - * The three fields are one indivisible unit: the backend reads the count only - * relative to the watermark, and returns its inline delta only relative to the - * cursor. Passing them as a single object is what keeps them from drifting - * apart at a call site. + * On a slot-numbered run this is `eventCount`, optionally with the set of + * correlation ids the writer is blocked on. On a ULID-numbered run it is the + * `stateUpdatedAt` / `stateEventCount` / `stateCursor` triple, whose three + * fields are one indivisible unit: the backend reads the count only relative to + * the watermark, and returns its inline delta only relative to the cursor. + * Passing them as a single object is what keeps them from drifting apart at a + * call site. */ export interface PreconditionSnapshotParams { stateUpdatedAt?: number; stateEventCount?: number; stateCursor?: string; + eventCount?: number; } /** @@ -809,6 +1014,14 @@ export function preconditionSnapshotParams( if (!isPreconditionGuardEnabled()) { return {}; } + // A slot-numbered run says with one integer everything the triple was + // approximating, so the two are alternatives rather than a pair. Sending the + // triple here would also be futile: a slot id carries no time, so + // `latestEventStateUpdatedAt` would fail open on every single write. + const eventCount = maxEventSlot(events); + if (eventCount !== undefined) { + return { eventCount }; + } const stateUpdatedAt = latestEventStateUpdatedAt(events); if (stateUpdatedAt === undefined) { return {}; @@ -865,6 +1078,12 @@ export function preconditionEventDelta( }; } +/** Creates one event on a bound run, carrying replay-recovery telemetry. */ +export type EventCreator = ( + data: CreateEventRequest, + params?: CreateEventParams +) => Promise; + /** * CORS headers for health check responses. * Allows the observability UI to check endpoint health from a different origin. diff --git a/packages/core/src/runtime/precondition-guard-replay.test.ts b/packages/core/src/runtime/precondition-guard-replay.test.ts index 49769f2518..fefa5a0bcf 100644 --- a/packages/core/src/runtime/precondition-guard-replay.test.ts +++ b/packages/core/src/runtime/precondition-guard-replay.test.ts @@ -37,7 +37,6 @@ import { dehydrateStepReturnValue, dehydrateWorkflowArguments, } from '../serialization.js'; -import { pinSharedCorrelationIds } from '../test-support/correlation-id-scheme.js'; import { createContext } from '../vm/index.js'; import { getPreconditionMaxInProcessRestarts, @@ -931,8 +930,6 @@ async function inlineClaimRejectionScenario() { }; } -pinSharedCorrelationIds(); - describe('precondition guard through the real replay loop', () => { let originalGuard: string | undefined; let originalRestartBound: string | undefined; diff --git a/packages/core/src/runtime/resume-hook.consumer-preload.test.ts b/packages/core/src/runtime/resume-hook.consumer-preload.test.ts index 1e8473fd6e..52ee1ca227 100644 --- a/packages/core/src/runtime/resume-hook.consumer-preload.test.ts +++ b/packages/core/src/runtime/resume-hook.consumer-preload.test.ts @@ -44,7 +44,6 @@ import { dehydrateStepReturnValue, dehydrateWorkflowArguments, } from '../serialization.js'; -import { pinSharedCorrelationIds } from '../test-support/correlation-id-scheme.js'; import { createContext } from '../vm/index.js'; import { setWorld } from './world.js'; @@ -409,8 +408,6 @@ async function runResumeConsumerScenario(options: { }; } -pinSharedCorrelationIds(); - describe('lazy hook resume consumer preload', () => { afterEach(() => { setWorld(undefined); diff --git a/packages/core/src/runtime/start.test.ts b/packages/core/src/runtime/start.test.ts index 8904600556..28a62755ba 100644 --- a/packages/core/src/runtime/start.test.ts +++ b/packages/core/src/runtime/start.test.ts @@ -2,8 +2,10 @@ import { WorkflowRuntimeError, WorkflowWorldError } from '@workflow/errors'; import { SPEC_VERSION_CURRENT, SPEC_VERSION_LEGACY, + SPEC_VERSION_MAX_SUPPORTED, SPEC_VERSION_SUPPORTS_ATTRIBUTES, SPEC_VERSION_SUPPORTS_CBOR_QUEUE_TRANSPORT, + SPEC_VERSION_SUPPORTS_SLOT_IDENTITY, } from '@workflow/world'; import { afterEach, @@ -136,7 +138,7 @@ describe('start', () => { } as any); await expect(start(validWorkflow, [])).rejects.toThrow( - 'requires a World with matching spec version' + 'supports Worlds with spec version' ); expect(mockEventsCreate).not.toHaveBeenCalled(); expect(mockQueue).not.toHaveBeenCalled(); @@ -174,7 +176,7 @@ describe('start', () => { } as any); await expect(start(validWorkflow, [])).rejects.toThrow( - 'requires a World with matching spec version' + 'supports Worlds with spec version' ); expect(mockEventsCreate).not.toHaveBeenCalled(); expect(mockQueue).not.toHaveBeenCalled(); @@ -186,19 +188,49 @@ describe('start', () => { }); setWorld({ - specVersion: SPEC_VERSION_CURRENT + 1, + specVersion: SPEC_VERSION_MAX_SUPPORTED + 1, getDeploymentId: vi.fn().mockResolvedValue('deploy_123'), events: { create: mockEventsCreate }, queue: mockQueue, } as any); await expect(start(validWorkflow, [])).rejects.toThrow( - 'requires a World with matching spec version' + 'supports Worlds with spec version' ); expect(mockEventsCreate).not.toHaveBeenCalled(); expect(mockQueue).not.toHaveBeenCalled(); }); + it('accepts a world that opts into a spec version above the default', async () => { + // `world-vercel` declares the slot-identity version so its new runs are + // created with slot event ids. An equality check against the default + // would make the runtime refuse the adapter shipped alongside it, and + // the failure surfaces only in e2e against that World. + const validWorkflow = Object.assign(() => Promise.resolve('result'), { + workflowId: 'test-workflow', + }); + + setWorld({ + specVersion: SPEC_VERSION_SUPPORTS_SLOT_IDENTITY, + getDeploymentId: vi.fn().mockResolvedValue('deploy_123'), + events: { create: mockEventsCreate }, + queue: mockQueue, + } as any); + + await start(validWorkflow, []); + + // The declared version is what gets stamped on `run_created`, which is + // what pins the run's id scheme for the rest of its life. + expect(mockEventsCreate).toHaveBeenCalledWith( + expect.stringMatching(/^wrun_/), + expect.objectContaining({ + eventType: 'run_created', + specVersion: SPEC_VERSION_SUPPORTS_SLOT_IDENTITY, + }), + expect.anything() + ); + }); + it('should use provided specVersion when passed in options', async () => { const validWorkflow = Object.assign(() => Promise.resolve('result'), { workflowId: 'test-workflow', diff --git a/packages/core/src/runtime/suspension-handler.test.ts b/packages/core/src/runtime/suspension-handler.test.ts index e63beaf297..6a6e883f7c 100644 --- a/packages/core/src/runtime/suspension-handler.test.ts +++ b/packages/core/src/runtime/suspension-handler.test.ts @@ -4,9 +4,11 @@ import { PreconditionFailedError, WorkflowWorldError, } from '@workflow/errors'; -import type { WorkflowRun, World } from '@workflow/world'; +import type { Event, WorkflowRun, World } from '@workflow/world'; +import { slotToEventId } from '@workflow/world'; import { describe, expect, it, vi } from 'vitest'; import { WorkflowSuspension } from '../global.js'; +import { maxEventSlot } from './helpers.js'; import { ReplayRecoveryReporter } from './replay-recovery-reporter.js'; import { handleSuspension } from './suspension-handler.js'; @@ -542,6 +544,82 @@ describe('handleSuspension', () => { }) ).rejects.toBeInstanceOf(PreconditionFailedError); }); + + describe('skipped-slot reports', () => { + /** A slot-numbered log event, minimal beyond what a snapshot reads. */ + function slotEvent(slot: number, eventType: Event['eventType']): Event { + return { + eventId: slotToEventId(slot), + eventType, + runId: run.runId, + createdAt: new Date(), + } as Event; + } + + /** One wait, so exactly one guarded write carries the report back. */ + function oneWait() { + return new Map([ + [ + 'wait_reported', + { + type: 'wait' as const, + correlationId: 'wait_reported', + resumeAt: new Date(Date.now() + 60_000), + }, + ], + ]); + } + + it('merges a complete report into the caller event log', async () => { + const eventLog = { events: [slotEvent(1, 'run_started')], cursor: null }; + const skipped = slotEvent(2, 'hook_received'); + const eventsCreate = vi.fn(async (_runId, event) => ({ + event: { ...event, eventId: slotToEventId(3) }, + events: [skipped], + })); + + const result = await handleSuspension({ + suspension: new WorkflowSuspension(oneWait(), globalThis), + world: createWorld(eventsCreate), + run, + eventLog, + }); + + expect(result.reportedEventCount).toBe(1); + // The replay that resumes from this log sees the skipped event without + // reloading, and the log still says how far it reaches. + expect(eventLog.events.map((e) => e.eventId)).toEqual([ + slotToEventId(1), + slotToEventId(2), + ]); + expect(maxEventSlot(eventLog.events)).toBe(2); + }); + + it('drops a truncated report instead of raising the log past a hole', async () => { + const eventLog = { events: [slotEvent(1, 'run_started')], cursor: null }; + // Slot 2 is on the same skipped span but absent from the report, so + // merging slot 3 would put the log's maximum above a missing position. + // Later writes read that maximum to say what they have seen, and a World + // only reports the span a write skips, so slot 2 would never be sent. + const skipped = slotEvent(3, 'hook_received'); + const eventsCreate = vi.fn(async (_runId, event) => ({ + event: { ...event, eventId: slotToEventId(4) }, + events: [skipped], + hasMore: true, + })); + + const result = await handleSuspension({ + suspension: new WorkflowSuspension(oneWait(), globalThis), + world: createWorld(eventsCreate), + run, + eventLog, + }); + + expect(result.reportedEventCount).toBe(0); + expect(eventLog.events.map((e) => e.eventId)).toEqual([slotToEventId(1)]); + expect(maxEventSlot(eventLog.events)).toBe(1); + }); + }); }); describe('retainedStepInputsSafe (serialization passivity gate)', () => { diff --git a/packages/core/src/runtime/suspension-handler.ts b/packages/core/src/runtime/suspension-handler.ts index 3d35954d03..b02826df4c 100644 --- a/packages/core/src/runtime/suspension-handler.ts +++ b/packages/core/src/runtime/suspension-handler.ts @@ -32,7 +32,12 @@ import { dehydrateStepArguments } from '../serialization.js'; import * as Attribute from '../telemetry/semantic-conventions.js'; import { getAbortStreamIdFromToken } from '../util.js'; import { getMaxInlineSteps } from './constants.js'; -import { type LoadedEventLog, preconditionSnapshotParams } from './helpers.js'; +import { + type EventCreator, + type LoadedEventLog, + mergeReportedEvents, + preconditionSnapshotParams, +} from './helpers.js'; import { ReplayRecoveryReporter } from './replay-recovery-reporter.js'; export interface SuspensionHandlerParams { @@ -81,6 +86,13 @@ export interface SuspensionHandlerResult { * into the same batch boundary. */ createdStepCorrelationIds: Set; + /** + * How many events this phase's writes reported back as occupying slots they + * skipped over, already merged into the caller's `eventLog.events`. Nonzero + * means the array was reordered to restore slot order, so any index the + * caller cached into it (payload prewarm scan position) is stale. + */ + reportedEventCount: number; /** * The steps whose `step_created` writes were intentionally deferred so the * caller can run them inline via lazy `step_started` events (which create @@ -294,16 +306,52 @@ export async function handleSuspension({ // because the event's correlation id was minted by *this* replay's seeded // sequence, so re-committing it against a corrected log would persist an // event no correct replay produces. - const createGuarded = ( - data: CreateEventRequest, - params?: CreateEventParams - ) => - eventLog - ? createEvent(data, { - ...params, - ...preconditionSnapshotParams(eventLog.events, eventLog.cursor), - }) - : createEvent(data, params); + let reportedEvents = 0; + const createGuarded: EventCreator = async (data, params) => { + if (!eventLog) { + return createEvent(data, params); + } + const log = eventLog; + const result = await createEvent(data, { + ...params, + ...preconditionSnapshotParams(log.events, log.cursor), + }); + // Bump-and-report: the write landed above the slot it asked for, so these + // are the events it was decided without. Merging them here rather than at + // each call site means the rest of this phase's writes β€” which read the + // same array to build their own snapshot β€” ask for a slot above them, and + // the replay that resumes from this log sees them without a reload. + // + // A truncated report (`hasMore`) is dropped whole rather than merged, the + // same way the wait loop treats one. It covers a span of positions but + // carries only some of the events on them, so merging it would raise the + // log's highest position past a position whose event is missing. Every + // later write of this phase reads that maximum to say what it has seen, so + // each would claim a position it never saw and the World, which only + // reports the span a write skips, would never send it. Dropping the report + // costs one more round of the same events on the next write and keeps the + // log a prefix of the truth. + if (result.events?.length && result.hasMore !== true) { + const added = mergeReportedEvents(log.events, result.events); + reportedEvents += added; + if (added > 0) { + runtimeLogger.debug('Suspension write skipped occupied slots', { + workflowRunId: runId, + eventType: data.eventType, + eventId: result.event?.eventId, + reported: added, + }); + } + } else if (result.events?.length) { + runtimeLogger.debug('Dropped a truncated skipped-slot report', { + workflowRunId: runId, + eventType: data.eventType, + eventId: result.event?.eventId, + offered: result.events.length, + }); + } + return result; + }; // Separate queue items by type const stepItems = suspension.steps.filter( (item): item is StepInvocationQueueItem => item.type === 'step' @@ -816,6 +864,7 @@ export async function handleSuspension({ hasHookEvents: hooksNeedingCreation.length > 0, hookCreationMs, retainedStepInputsSafe, + reportedEventCount: reportedEvents, }; } diff --git a/packages/core/src/runtime/wait-completion-replay.test.ts b/packages/core/src/runtime/wait-completion-replay.test.ts index a215bab7d3..8da3149a36 100644 --- a/packages/core/src/runtime/wait-completion-replay.test.ts +++ b/packages/core/src/runtime/wait-completion-replay.test.ts @@ -4,6 +4,7 @@ import { type Event, type EventResult, SPEC_VERSION_CURRENT, + slotToEventId, type WorkflowRun, type World, } from '@workflow/world'; @@ -16,7 +17,6 @@ import { dehydrateStepReturnValue, dehydrateWorkflowArguments, } from '../serialization.js'; -import { pinSharedCorrelationIds } from '../test-support/correlation-id-scheme.js'; import { createContext } from '../vm/index.js'; import { setWorld } from './world.js'; @@ -81,6 +81,12 @@ async function runStaleWaitReplayScenario(options: { returnInlineDelta?: boolean; /** Truncate that inline delta (hasMore: true), which must not be absorbed. */ inlineDeltaHasMore?: boolean; + /** + * Number the fake log with slot event ids. That is what makes the handler + * send the slot precondition, so it is the only mode in which a write can + * carry both halves of the World's answer channel. + */ + slotEventIds?: boolean; }) { vi.spyOn(Date, 'now').mockReturnValue(+fixedNow); @@ -125,7 +131,9 @@ async function runStaleWaitReplayScenario(options: { ...data, specVersion: data.specVersion ?? SPEC_VERSION_CURRENT, runId, - eventId: `evt_${eventIndex.toString().padStart(3, '0')}`, + eventId: options.slotEventIds + ? slotToEventId(eventIndex) + : `evt_${eventIndex.toString().padStart(3, '0')}`, createdAt, }) as Event; @@ -453,6 +461,7 @@ async function runStaleWaitReplayScenario(options: { listEvents, listedPages, queue, + staleEvents, preloadedEvents, preloadedCursor, staleEventsCursor, @@ -489,8 +498,6 @@ function expectHookBranchQueued( ); } -pinSharedCorrelationIds(); - describe('workflow handler wait completion replay', () => { afterEach(() => { setWorld(undefined); @@ -684,6 +691,32 @@ describe('workflow handler wait completion replay', () => { expectHookBranchQueued(result); }); + it('asks a slot-numbered World for the delta and the skipped slots at once', async () => { + // On a slot-numbered run the write carries both halves of the World's + // answer channel: `sinceCursor` asks for the delta since the handler's + // snapshot, and `eventCount` states the slot that snapshot reached so a + // bumped write can report what it was decided without. They share + // `events`/`cursor`/`hasMore` on the response, so a World that answers + // both has to pick one, and the delta is the superset. Anything narrower + // returned alongside the delta's cursor loses the difference. + const result = await runStaleWaitReplayScenario({ + includePreloadedCursor: true, + returnInlineDelta: true, + slotEventIds: true, + }); + + const waitWrite = result.createEvent.mock.calls.find( + (call) => (call[1] as CreateEventRequest).eventType === 'wait_completed' + ); + expect(waitWrite?.[2]).toEqual( + expect.objectContaining({ + sinceCursor: result.staleEventsCursor, + eventCount: result.staleEvents.length, + }) + ); + expectHookBranchQueued(result); + }); + it('falls back to the follow-up fetch when the returned delta is truncated', async () => { // hasMore means the page is not the whole delta. Absorbing it would leave // a hole between the events taken and the cursor reported, so the handler diff --git a/packages/core/src/runtime/world-compatibility.ts b/packages/core/src/runtime/world-compatibility.ts index c06c26eefb..247ad73d68 100644 --- a/packages/core/src/runtime/world-compatibility.ts +++ b/packages/core/src/runtime/world-compatibility.ts @@ -1,19 +1,44 @@ import { WorkflowRuntimeError } from '@workflow/errors'; import type { World } from '@workflow/world'; -import { SPEC_VERSION_CURRENT } from '@workflow/world'; +import { + SPEC_VERSION_CURRENT, + SPEC_VERSION_MAX_SUPPORTED, +} from '@workflow/world'; type WorldSpecVersionMetadata = Pick; +/** + * Rejects a World this runtime cannot speak to. + * + * The accepted range is `[SPEC_VERSION_CURRENT, SPEC_VERSION_MAX_SUPPORTED]`. + * Below the current version means an old World package paired with a new + * runtime, which cannot serve the protocol this runtime speaks. Above the + * ceiling means a World built against a newer spec than this runtime knows how + * to read. + * + * The range has a floor and a ceiling rather than a single value because a + * World may opt into a spec version above the default: `world-vercel` declares + * the slot-identity version so its new runs are created with slot event ids, + * while every other World stays on the default. An equality check would make + * this runtime refuse the adapter shipped alongside it. + */ export function assertWorldSupportsRuntimeProtocol( world: WorldSpecVersionMetadata ): void { - if (world.specVersion === SPEC_VERSION_CURRENT) { + const declared = world.specVersion; + if ( + declared !== undefined && + declared !== null && + declared >= SPEC_VERSION_CURRENT && + declared <= SPEC_VERSION_MAX_SUPPORTED + ) { return; } - const supportedVersion = world.specVersion ?? 'none'; + const supportedVersion = declared ?? 'none'; throw new WorkflowRuntimeError( - `This Workflow runtime requires a World with matching spec version ${SPEC_VERSION_CURRENT}, ` + + `This Workflow runtime supports Worlds with spec version ${SPEC_VERSION_CURRENT} ` + + `through ${SPEC_VERSION_MAX_SUPPORTED}, ` + `but the configured World declares spec version ${supportedVersion}. ` + 'Install a World package version compatible with the current Workflow runtime.' ); diff --git a/packages/core/src/step-delivery-hop-count.test.ts b/packages/core/src/step-delivery-hop-count.test.ts index ff511bbcf9..7653f85d4e 100644 --- a/packages/core/src/step-delivery-hop-count.test.ts +++ b/packages/core/src/step-delivery-hop-count.test.ts @@ -29,7 +29,6 @@ import type { Event } from '@workflow/world'; import * as nanoid from 'nanoid'; import { monotonicFactory } from 'ulid'; import { describe, expect, it, vi } from 'vitest'; -import { createCorrelationIdGenerator } from './correlation-id.js'; import { EventsConsumer } from './events-consumer.js'; import { WorkflowSuspension } from './global.js'; import type { WorkflowOrchestratorContext } from './private.js'; @@ -62,6 +61,8 @@ function setupWorkflowContext( replayPayloadCache, globalThis: context.globalThis, eventsConsumer: new EventsConsumer(events, { + // Fake context: no deliveries are modeled, so the gate is a no-op here. + isDeliveryIdle: () => true, onUnconsumedEvent: (event) => { ctxRef.current?.onWorkflowError( new WorkflowRuntimeError(`Unconsumed event: ${event.eventType}`) @@ -70,14 +71,7 @@ function setupWorkflowContext( getPromiseQueue: () => promiseQueueHolder.current, }), invocationsQueue: new Map(), - generateCorrelationId: createCorrelationIdGenerator({ - seed: 'test', - fixedTimestamp: workflowStartedAt, - positional: () => ulid(workflowStartedAt), - // The event logs in this file hardcode correlation ids the run-wide - // shared sequence minted, so replay only matches under that scheme. - perKind: false, - }), + generateUlid: () => ulid(workflowStartedAt), generateNanoid: nanoid.customRandom(nanoid.urlAlphabet, 21, (size) => new Uint8Array(size).map(() => 256 * context.globalThis.Math.random()) ), diff --git a/packages/core/src/step-delivery-ordering.test.ts b/packages/core/src/step-delivery-ordering.test.ts index 60213522f8..300cb6dba4 100644 --- a/packages/core/src/step-delivery-ordering.test.ts +++ b/packages/core/src/step-delivery-ordering.test.ts @@ -4,7 +4,6 @@ import type { Event } from '@workflow/world'; import * as nanoid from 'nanoid'; import { monotonicFactory } from 'ulid'; import { afterEach, describe, expect, it, vi } from 'vitest'; -import { createCorrelationIdGenerator } from './correlation-id.js'; import { EventsConsumer } from './events-consumer.js'; import { WorkflowSuspension } from './global.js'; import type { WorkflowOrchestratorContext } from './private.js'; @@ -109,6 +108,8 @@ function setupWorkflowContext( replayPayloadCache, globalThis: context.globalThis, eventsConsumer: new EventsConsumer(events, { + // Fake context: no deliveries are modeled, so the gate is a no-op here. + isDeliveryIdle: () => true, onUnconsumedEvent: (event) => { ctxRef.current?.onWorkflowError( new WorkflowRuntimeError( @@ -119,14 +120,7 @@ function setupWorkflowContext( getPromiseQueue: () => promiseQueueHolder.current, }), invocationsQueue: new Map(), - generateCorrelationId: createCorrelationIdGenerator({ - seed: 'test', - fixedTimestamp: workflowStartedAt, - positional: () => ulid(workflowStartedAt), - // The event logs in this file hardcode correlation ids the run-wide - // shared sequence minted, so replay only matches under that scheme. - perKind: false, - }), + generateUlid: () => ulid(workflowStartedAt), generateNanoid: nanoid.customRandom(nanoid.urlAlphabet, 21, (size) => new Uint8Array(size).map(() => 256 * context.globalThis.Math.random()) ), diff --git a/packages/core/src/step-hydration-memoization.test.ts b/packages/core/src/step-hydration-memoization.test.ts index 8708c02e5d..e8aabc1d0d 100644 --- a/packages/core/src/step-hydration-memoization.test.ts +++ b/packages/core/src/step-hydration-memoization.test.ts @@ -2,7 +2,6 @@ import type { Event } from '@workflow/world'; import * as nanoid from 'nanoid'; import { monotonicFactory } from 'ulid'; import { afterEach, describe, expect, it, vi } from 'vitest'; -import { createCorrelationIdGenerator } from './correlation-id.js'; import { EventsConsumer } from './events-consumer.js'; import type { WorkflowOrchestratorContext } from './private.js'; import { ReplayPayloadCache } from './replay-payload-cache.js'; @@ -39,18 +38,13 @@ function setupWorkflowContext( encryptionKey: undefined, globalThis: context.globalThis, eventsConsumer: new EventsConsumer(events, { + // Fake context: no deliveries are modeled, so the gate is a no-op here. + isDeliveryIdle: () => true, onUnconsumedEvent: () => {}, getPromiseQueue: () => Promise.resolve(), }), invocationsQueue: new Map(), - generateCorrelationId: createCorrelationIdGenerator({ - seed: 'test', - fixedTimestamp: workflowStartedAt, - positional: () => ulid(workflowStartedAt), - // The event logs in this file hardcode correlation ids the run-wide - // shared sequence minted, so replay only matches under that scheme. - perKind: false, - }), + generateUlid: () => ulid(workflowStartedAt), generateNanoid: nanoid.customRandom(nanoid.urlAlphabet, 21, (size) => new Uint8Array(size).map(() => 256 * context.globalThis.Math.random()) ), diff --git a/packages/core/src/step.test.ts b/packages/core/src/step.test.ts index faf852a54b..574e441110 100644 --- a/packages/core/src/step.test.ts +++ b/packages/core/src/step.test.ts @@ -9,7 +9,6 @@ import * as nanoid from 'nanoid'; import { monotonicFactory } from 'ulid'; import { beforeAll, describe, expect, it, vi } from 'vitest'; import { registerSerializationClass } from './class-serialization.js'; -import { createCorrelationIdGenerator } from './correlation-id.js'; import { EventsConsumer } from './events-consumer.js'; import { WorkflowSuspension } from './global.js'; import type { WorkflowOrchestratorContext } from './private.js'; @@ -58,18 +57,13 @@ function setupWorkflowContext(events: Event[]): WorkflowOrchestratorContext { replayPayloadCache: new ReplayPayloadCache(undefined), globalThis: context.globalThis, eventsConsumer: new EventsConsumer(events, { + // Fake context: no deliveries are modeled, so the gate is a no-op here. + isDeliveryIdle: () => true, onUnconsumedEvent: () => {}, getPromiseQueue: () => Promise.resolve(), }), invocationsQueue: new Map(), - generateCorrelationId: createCorrelationIdGenerator({ - seed: 'test', - fixedTimestamp: workflowStartedAt, - positional: () => ulid(workflowStartedAt), - // The event logs in this file hardcode correlation ids the run-wide - // shared sequence minted, so replay only matches under that scheme. - perKind: false, - }), + generateUlid: () => ulid(workflowStartedAt), generateNanoid: nanoid.customRandom(nanoid.urlAlphabet, 21, (size) => new Uint8Array(size).map(() => 256 * context.globalThis.Math.random()) ), diff --git a/packages/core/src/step.ts b/packages/core/src/step.ts index d40bcdb19b..1fcc4c11df 100644 --- a/packages/core/src/step.ts +++ b/packages/core/src/step.ts @@ -25,7 +25,7 @@ export function createUseStep(ctx: WorkflowOrchestratorContext) { ): Promise { const { promise, resolve, reject } = withResolvers(); - const correlationId = `step_${ctx.generateCorrelationId('step')}`; + const correlationId = `step_${ctx.generateUlid()}`; const queueItem: StepInvocationQueueItem = { type: 'step', diff --git a/packages/core/src/telemetry/semantic-conventions.ts b/packages/core/src/telemetry/semantic-conventions.ts index 56f8732eb8..f4fc7cbf4f 100644 --- a/packages/core/src/telemetry/semantic-conventions.ts +++ b/packages/core/src/telemetry/semantic-conventions.ts @@ -82,6 +82,32 @@ export const WorkflowExecutionMode = SemanticConvention<'replay' | 'retained'>( 'workflow.execution.mode' ); +/** + * Events the replay walked past that no consumer claimed, still held when the + * replay stopped. + * + * A non-zero count on a suspension is ordinary: an out-of-band delivery that + * landed ahead of the code that reads it waits for the replay that gets there. + * From inside one replay that is indistinguishable from an event no replay will + * ever claim, because the two differ only in what the next replay does. So the + * count goes on the span instead of failing the run, and the case worth acting + * on is a query across a run's spans: the same + * {@link WorkflowParkedEventId} held on suspension after suspension. + */ +export const WorkflowParkedEventsCount = SemanticConvention( + 'workflow.events.parked.count' +); + +/** Oldest event still held unclaimed when the replay stopped. */ +export const WorkflowParkedEventId = SemanticConvention( + 'workflow.events.parked.event_id' +); + +/** Type of the oldest event still held unclaimed when the replay stopped. */ +export const WorkflowParkedEventType = SemanticConvention( + 'workflow.events.parked.event_type' +); + /** Number of arguments passed to the workflow */ export const WorkflowArgumentsCount = SemanticConvention( 'workflow.arguments.count' diff --git a/packages/core/src/test-support/correlation-id-scheme.ts b/packages/core/src/test-support/correlation-id-scheme.ts deleted file mode 100644 index 32a000dfdd..0000000000 --- a/packages/core/src/test-support/correlation-id-scheme.ts +++ /dev/null @@ -1,26 +0,0 @@ -import { afterAll, beforeAll } from 'vitest'; - -/** - * Pins a test file to the run-wide shared correlation-id sequence. - * - * Replay tests that drive the real `workflowEntrypoint` against an event log - * with hardcoded correlation ids can only match under the scheme those ids were - * minted by, and the fixtures in this repo predate per-kind sequences. Files - * whose fixture ids are derived rather than written out run under whichever - * scheme `WORKFLOW_PER_KIND_CORRELATION_IDS` selects; per-kind minting itself is - * covered by `correlation-id.test.ts`. - */ -export function pinSharedCorrelationIds(): void { - let original: string | undefined; - beforeAll(() => { - original = process.env.WORKFLOW_PER_KIND_CORRELATION_IDS; - process.env.WORKFLOW_PER_KIND_CORRELATION_IDS = '0'; - }); - afterAll(() => { - if (original === undefined) { - delete process.env.WORKFLOW_PER_KIND_CORRELATION_IDS; - } else { - process.env.WORKFLOW_PER_KIND_CORRELATION_IDS = original; - } - }); -} diff --git a/packages/core/src/unconsumed-check-delivery-idle.test.ts b/packages/core/src/unconsumed-check-delivery-idle.test.ts index 8a13f74dd8..5acde2631b 100644 --- a/packages/core/src/unconsumed-check-delivery-idle.test.ts +++ b/packages/core/src/unconsumed-check-delivery-idle.test.ts @@ -1,7 +1,11 @@ import { withResolvers } from '@workflow/utils'; import type { Event } from '@workflow/world'; import { afterEach, describe, expect, it, vi } from 'vitest'; -import { EventConsumerResult, EventsConsumer } from './events-consumer.js'; +import { + EventConsumerResult, + EventsConsumer, + MIN_DEFERRED_CHECK_DELAY_MS, +} from './events-consumer.js'; import type { WorkflowOrchestratorContext } from './private.js'; import { isDeliveryIdle, registerDeliveryBarrier } from './private.js'; @@ -17,11 +21,15 @@ import { isDeliveryIdle, registerDeliveryBarrier } from './private.js'; * The unconsumed-event check used to resolve that by waiting a fixed * `DEFERRED_CHECK_DELAY_MS` after the promise queue drained, which is a bet * that every delivery lands inside the window. Replaying a batch of N parallel - * step results loses it: the queue drains with N-1 of them still on the - * detached path, and the check declares `ReplayDivergenceError` against a log - * the very same replay goes on to reproduce exactly. Measured on the event-log - * race repro against world-postgres, on identical event logs: 0 of 114 runs - * corrupted with a 100ms window, 34 of 42 with a 10ms one. + * step results is what puts that bet under load: the queue drains with N-1 of + * them still on the detached path, so whether the check declares + * `ReplayDivergenceError` against a log the very same replay goes on to + * reproduce exactly comes down to the clock. Shrinking the window makes it lose. + * Measured on the event-log race repro against world-postgres, on identical + * event logs: 0 of 114 runs corrupted at the 100ms default, 34 of 42 at 10ms. + * That is evidence for the mechanism, not for the default being too short; the + * delay is also a user-settable override, so the old behaviour stayed one + * configuration away from losing. * * `hasParkedCommittedDelivery` in private.ts already documents this hazard for * the suspension path (vercel/workflow#3183). These tests pin the same guard @@ -30,6 +38,17 @@ import { isDeliveryIdle, registerDeliveryBarrier } from './private.js'; * and the check must still fire for an event no delivery is waiting on. */ +/** + * Shortest delay the check accepts, and a wait comfortably past it. Both derive + * from the floor so that raising the floor cannot quietly turn the negative + * assertions below into no-ops: were the stub a hardcoded number the floor + * outgrew, `getDeferredCheckDelayMs` would clamp it up, the delay would stop + * being shorter than the delivery, and the check would be "not fired yet" + * rather than "held off by the gate". + */ +const CHECK_DELAY_MS = MIN_DEFERRED_CHECK_DELAY_MS; +const PAST_CHECK_DELAY_MS = CHECK_DELAY_MS * 25; + function createEvent(overrides: Partial = {}): Event { return { id: 'event-1', @@ -69,7 +88,7 @@ describe('unconsumed-event check against in-flight deliveries', () => { it('does not declare divergence while a step delivery is outstanding', async () => { // Far shorter than the delivery below, so the run survives only if the // check waits for the delivery rather than for the clock. - vi.stubEnv('WORKFLOW_DEFERRED_CHECK_DELAY_MS', '10'); + vi.stubEnv('WORKFLOW_DEFERRED_CHECK_DELAY_MS', String(CHECK_DELAY_MS)); const ctx = createDeliveryContext(); const event = createEvent(); @@ -88,7 +107,7 @@ describe('unconsumed-event check against in-flight deliveries', () => { vi.fn().mockReturnValue(EventConsumerResult.NotConsumed) ); - await new Promise((resolve) => setTimeout(resolve, 250)); + await new Promise((resolve) => setTimeout(resolve, PAST_CHECK_DELAY_MS)); expect(onUnconsumedEvent).not.toHaveBeenCalled(); // The delivery lands and the workflow reaches the call this event records. @@ -98,12 +117,12 @@ describe('unconsumed-event check against in-flight deliveries', () => { await vi.waitFor(() => { expect(consumer.eventIndex).toBe(1); }); - await new Promise((resolve) => setTimeout(resolve, 50)); + await new Promise((resolve) => setTimeout(resolve, PAST_CHECK_DELAY_MS)); expect(onUnconsumedEvent).not.toHaveBeenCalled(); }); it('does not declare divergence while a payload is hydrating', async () => { - vi.stubEnv('WORKFLOW_DEFERRED_CHECK_DELAY_MS', '10'); + vi.stubEnv('WORKFLOW_DEFERRED_CHECK_DELAY_MS', String(CHECK_DELAY_MS)); const ctx = createDeliveryContext(); const event = createEvent(); @@ -121,7 +140,7 @@ describe('unconsumed-event check against in-flight deliveries', () => { vi.fn().mockReturnValue(EventConsumerResult.NotConsumed) ); - await new Promise((resolve) => setTimeout(resolve, 250)); + await new Promise((resolve) => setTimeout(resolve, PAST_CHECK_DELAY_MS)); expect(onUnconsumedEvent).not.toHaveBeenCalled(); ctx.pendingDeliveries--; diff --git a/packages/core/src/workflow.test.ts b/packages/core/src/workflow.test.ts index fdeb2bd97a..f47e1d1d7e 100644 --- a/packages/core/src/workflow.test.ts +++ b/packages/core/src/workflow.test.ts @@ -13,15 +13,12 @@ import { dehydrateWorkflowArguments, hydrateWorkflowReturnValue, } from './serialization.js'; -import { pinSharedCorrelationIds } from './test-support/correlation-id-scheme.js'; import { createContext } from './vm/index.js'; import { replayWorkflow, resumeWorkflow, runWorkflow } from './workflow.js'; // No encryption key = encryption disabled const noEncryptionKey = undefined; -pinSharedCorrelationIds(); - describe('runWorkflow', () => { const getWorkflowTransformCode = (workflowName?: string) => `;globalThis.__private_workflows = new Map(); @@ -382,13 +379,16 @@ describe('runWorkflow', () => { assert(suspended.type === 'suspended'); // A strict extension whose appended suffix the VM cannot consume: the - // resume starts, then diverges mid-execution. + // resume starts, then diverges mid-execution. It has to be a + // replay-origin type: the consumer walks past an unclaimed delivery and + // holds it for a later consumer, so only an event whose position is a + // replay's own decision record diverges on the spot. const alien = { eventId: 'event-alien', runId: run.runId, - eventType: 'hook_received', - correlationId: 'hook_unknown', - eventData: {}, + eventType: 'step_created', + correlationId: 'step_unknown', + eventData: { stepName: 'unknown' }, createdAt: new Date('2024-01-01T00:00:01.000Z'), } as Event; await expect(resumeWorkflow(suspended.session, [alien])).rejects.toThrow( @@ -406,7 +406,7 @@ describe('runWorkflow', () => { // Turbo's first delivery synthesizes `startedAt` from the local clock, // while later (non-turbo) deliveries load the server-canonical `startedAt`. // Replay matching must NOT depend on `startedAt`: correlation IDs come from - // `generateCorrelationId`, keyed off the run-ID-recovered `fixedTimestamp`, not + // `generateUlid`, keyed off the run-ID-recovered `fixedTimestamp`, not // `startedAt`. Here the recorded `add` event uses the createdAt-derived // correlation ID, but `startedAt` is months away β€” replay must still // regenerate the same ID and consume the completion rather than throwing diff --git a/packages/core/src/workflow.ts b/packages/core/src/workflow.ts index a6e1b0672b..17cf8ea855 100644 --- a/packages/core/src/workflow.ts +++ b/packages/core/src/workflow.ts @@ -15,10 +15,6 @@ import type { Event, WorkflowRun, WorldCapabilities } from '@workflow/world'; import { SPEC_VERSION_SUPPORTS_COMPRESSION } from '@workflow/world'; import * as nanoid from 'nanoid'; import { monotonicFactory } from 'ulid'; -import { - createCorrelationIdGenerator, - isPerKindCorrelationIdsEnabled, -} from './correlation-id.js'; import { EventConsumerResult, EventsConsumer } from './events-consumer.js'; import type { QueueItem } from './global.js'; import { ENOTSUP, WorkflowSuspension } from './global.js'; @@ -155,6 +151,16 @@ export type WorkflowResult = readonly type: 'suspended'; readonly suspension: WorkflowSuspension; readonly session: WorkflowSession; + /** + * Events the replay walked past unclaimed and is still holding, if any. + * Ordinary on a suspension, and only actionable across a run's + * suspensions, so it is reported for telemetry rather than acted on here. + */ + readonly parked?: { + readonly count: number; + readonly eventId: string; + readonly eventType: string; + }; }; /** @@ -238,6 +244,20 @@ function recordResult( }); } else if (span) { applyWorkflowSuspensionToSpan(result.suspension, span); + // Events this pass walked past unclaimed and is still holding. Ordinary on + // a suspension: an out-of-band delivery that landed ahead of the code that + // reads it waits for the pass that reaches that code, and failing here + // would fail exactly the runs that tolerance exists for. The case that is + // not ordinary β€” the same event still held pass after pass β€” is a shape + // across these spans, which is why the eventId is on each one and no pass + // tries to rule on it alone. + if (result.parked) { + span.setAttributes({ + ...Attribute.WorkflowParkedEventsCount(result.parked.count), + ...Attribute.WorkflowParkedEventId(result.parked.eventId), + ...Attribute.WorkflowParkedEventType(result.parked.eventType), + }); + } } return result; } @@ -325,14 +345,12 @@ async function createWorkflowSession({ : `http://localhost:${(await getPortLazy()) ?? 3000}` ); - const seed = `${workflowRun.runId}:${workflowRun.workflowName}:${workflowRun.deploymentId}`; - const { context, globalThis: vmGlobalThis, updateTimestamp, } = createContext({ - seed, + seed: `${workflowRun.runId}:${workflowRun.workflowName}:${workflowRun.deploymentId}`, fixedTimestamp, }); @@ -371,14 +389,9 @@ async function createWorkflowSession({ }; const ulid = monotonicFactory(() => vmGlobalThis.Math.random()); - const generateCorrelationId = createCorrelationIdGenerator({ - seed, - fixedTimestamp, - // Correlation IDs must be replay-stable. `startedAt` differs between a - // turbo delivery and a later server-backed replay, so use fixedTimestamp. - positional: () => ulid(fixedTimestamp), - perKind: isPerKindCorrelationIdsEnabled(), - }); + // Correlation IDs must be replay-stable. `startedAt` differs between a turbo + // delivery and a later server-backed replay, so use fixedTimestamp. + const generateUlid = () => ulid(fixedTimestamp); const generateNanoid = nanoid.customRandom(nanoid.urlAlphabet, 21, (size) => new Uint8Array(size).map(() => 256 * vmGlobalThis.Math.random()) ); @@ -393,9 +406,20 @@ async function createWorkflowSession({ // is before any delivery can be registered against it. const deliveryIdleHolder = { current: (): boolean => true }; + // The VM clock only ever moves forward. Consumption order is log order for + // everything whose order the replay decides, but an event the consumer + // parked is delivered after the walk has already passed events written after + // it, and letting its `createdAt` set the clock would make `Date.now()` go + // backwards inside a single replay. + let clock = fixedTimestamp; + const eventsConsumer = new EventsConsumer(events, { onConsumedEvent: (event) => { - updateTimestamp(+event.createdAt); + const at = +event.createdAt; + if (at > clock) { + clock = at; + updateTimestamp(at); + } }, onUnconsumedEvent: (event) => { onWorkflowError( @@ -416,7 +440,7 @@ async function createWorkflowSession({ globalThis: vmGlobalThis, onWorkflowError, eventsConsumer, - generateCorrelationId, + generateUlid, generateNanoid, invocationsQueue: new Map(), // Use getter/setter so the EventsConsumer's getPromiseQueue() always @@ -497,11 +521,11 @@ async function createWorkflowSession({ // Serialization mints stream ids through this symbol, and calls it with no // seed time. `monotonicFactory` returns `encodeTime(lastTime)` on its // increment branch, so one such call latches the *host* wall clock into - // `lastTime` and every id the run mints afterwards carries that timestamp - // instead of `fixedTimestamp` β€” a value that differs on every replay. - // Binding the seed time here keeps the whole run on one replay-stable clock. + // `lastTime`, and every id the run mints afterwards carries that timestamp + // instead of `fixedTimestamp`, a value that differs on every replay. Binding + // the seed time here keeps the whole run on one replay-stable clock. // @ts-expect-error - `@types/node` says symbol is not valid, but it does work - vmGlobalThis[STABLE_ULID] = () => generateCorrelationId('stream'); + vmGlobalThis[STABLE_ULID] = generateUlid; // Workflow code must import the deterministic `fetch` step from `workflow`. vmGlobalThis.fetch = () => { @@ -1066,12 +1090,34 @@ async function createWorkflowSession({ result = await Promise.race([workflowBody, interruption.promise]); } catch (error) { if (state.type === 'suspended' && error === state.suspension) { - return { type: 'suspended', suspension: state.suspension, session }; + return { + type: 'suspended', + suspension: state.suspension, + session, + // A suspension is not a settling point: the consumer for something + // held may well be registered by the replay that follows this one. + // So it is carried out for the span instead of being judged here. + parked: eventsConsumer.parkedSummary, + }; } return failWorkflow(error); } state = { type: 'completed' }; + // The consumer walks past events whose type carries no ordering claim and + // holds them for a consumer it expects a later `subscribe()` to register. + // The workflow function returning is the point where that expectation is + // settled: nothing more will subscribe, so anything still held was never + // anyone's, and completing here would drop it silently. + const stranded = eventsConsumer.strandedEvent; + if (stranded) { + return failWorkflow( + new ReplayDivergenceError( + `Replay finished without consuming event: eventType=${stranded.eventType}, correlationId=${stranded.correlationId}, eventId=${stranded.eventId}.`, + { eventId: stranded.eventId } + ) + ); + } try { const output = await dehydrateWorkflowReturnValue( result, diff --git a/packages/core/src/workflow/abort-controller.ts b/packages/core/src/workflow/abort-controller.ts index 3d94dee210..4d5d9a08af 100644 --- a/packages/core/src/workflow/abort-controller.ts +++ b/packages/core/src/workflow/abort-controller.ts @@ -111,7 +111,7 @@ export function createCreateAbortController(ctx: WorkflowOrchestratorContext) { readonly [ABORT_HOOK_TOKEN]: string; constructor() { - const id = ctx.generateCorrelationId('abort'); + const id = ctx.generateUlid(); const streamName = getAbortStreamId(id); const hookToken = `abrt_${id}`; @@ -120,10 +120,8 @@ export function createCreateAbortController(ctx: WorkflowOrchestratorContext) { this.signal = new WorkflowAbortSignal(streamName, hookToken); // Register an internal system hook in the invocations queue. - // isSystem prevents token namespace conflicts with user hooks. The id - // draws from its own family, not `hook`, so constructing an abort - // controller does not renumber hooks the workflow creates later. - const correlationId = `hook_${ctx.generateCorrelationId('abortHook')}`; + // isSystem prevents token namespace conflicts with user hooks. + const correlationId = `hook_${ctx.generateUlid()}`; ctx.invocationsQueue.set(correlationId, { type: 'hook', correlationId, diff --git a/packages/core/src/workflow/attribute-dispatcher.ts b/packages/core/src/workflow/attribute-dispatcher.ts index 95ac76dc08..760dee8bbc 100644 --- a/packages/core/src/workflow/attribute-dispatcher.ts +++ b/packages/core/src/workflow/attribute-dispatcher.ts @@ -17,7 +17,7 @@ export function createSetAttributes(ctx: WorkflowOrchestratorContext) { options: { allowReservedAttributes?: boolean } = {} ): Promise { const { promise, resolve } = withResolvers(); - const correlationId = `attr_${ctx.generateCorrelationId('attr')}`; + const correlationId = `attr_${ctx.generateUlid()}`; const queueItem: AttributeInvocationQueueItem = { type: 'attribute', correlationId, diff --git a/packages/core/src/workflow/hook.test.ts b/packages/core/src/workflow/hook.test.ts index fbc2c8063e..bdaa3e5442 100644 --- a/packages/core/src/workflow/hook.test.ts +++ b/packages/core/src/workflow/hook.test.ts @@ -12,7 +12,6 @@ import { aliasSerializationClass, RUN_CLASS_ID, } from '../class-serialization.js'; -import { createCorrelationIdGenerator } from '../correlation-id.js'; import { EventsConsumer } from '../events-consumer.js'; import { WorkflowSuspension } from '../global.js'; import type { WorkflowOrchestratorContext } from '../private.js'; @@ -43,18 +42,13 @@ function setupWorkflowContext(events: Event[]): WorkflowOrchestratorContext { replayPayloadCache: new ReplayPayloadCache(undefined), globalThis: context.globalThis, eventsConsumer: new EventsConsumer(events, { + // Fake context: no deliveries are modeled, so the gate is a no-op here. + isDeliveryIdle: () => true, onUnconsumedEvent: () => {}, getPromiseQueue: () => Promise.resolve(), }), invocationsQueue: new Map(), - generateCorrelationId: createCorrelationIdGenerator({ - seed: 'test', - fixedTimestamp: workflowStartedAt, - positional: () => ulid(workflowStartedAt), - // The event logs in this file hardcode correlation ids the run-wide - // shared sequence minted, so replay only matches under that scheme. - perKind: false, - }), + generateUlid: () => ulid(workflowStartedAt), generateNanoid: nanoid.customRandom(nanoid.urlAlphabet, 21, (size) => new Uint8Array(size).map(() => 256 * context.globalThis.Math.random()) ), diff --git a/packages/core/src/workflow/hook.ts b/packages/core/src/workflow/hook.ts index e95998b858..d5d5bed8ec 100644 --- a/packages/core/src/workflow/hook.ts +++ b/packages/core/src/workflow/hook.ts @@ -96,7 +96,7 @@ export function createCreateHook(ctx: WorkflowOrchestratorContext) { } // Generate hook ID and token - const correlationId = `hook_${ctx.generateCorrelationId('hook')}`; + const correlationId = `hook_${ctx.generateUlid()}`; const token = options.token ?? ctx.generateNanoid(); const tokenRetentionUntil = options.experimental_minRetention === undefined diff --git a/packages/core/src/workflow/sleep.test.ts b/packages/core/src/workflow/sleep.test.ts index 5b2d2e6edf..1dd45b52ce 100644 --- a/packages/core/src/workflow/sleep.test.ts +++ b/packages/core/src/workflow/sleep.test.ts @@ -4,7 +4,6 @@ import type { Event } from '@workflow/world'; import * as nanoid from 'nanoid'; import { monotonicFactory } from 'ulid'; import { describe, expect, it, vi } from 'vitest'; -import { createCorrelationIdGenerator } from '../correlation-id.js'; import { EventsConsumer } from '../events-consumer.js'; import { WorkflowSuspension } from '../global.js'; import type { WorkflowOrchestratorContext } from '../private.js'; @@ -28,6 +27,8 @@ function setupWorkflowContext(events: Event[]): WorkflowOrchestratorContext { globalThis: context.globalThis, // ctx.onWorkflowError is accessed via closure β€” it's defined below on the same object eventsConsumer: new EventsConsumer(events, { + // Fake context: no deliveries are modeled, so the gate is a no-op here. + isDeliveryIdle: () => true, onUnconsumedEvent: (event) => { ctx.onWorkflowError( new ReplayDivergenceError( @@ -39,14 +40,7 @@ function setupWorkflowContext(events: Event[]): WorkflowOrchestratorContext { getPromiseQueue: () => Promise.resolve(), }), invocationsQueue: new Map(), - generateCorrelationId: createCorrelationIdGenerator({ - seed: 'test', - fixedTimestamp: workflowStartedAt, - positional: () => ulid(workflowStartedAt), - // The event logs in this file hardcode correlation ids the run-wide - // shared sequence minted, so replay only matches under that scheme. - perKind: false, - }), + generateUlid: () => ulid(workflowStartedAt), generateNanoid: nanoid.customRandom(nanoid.urlAlphabet, 21, (size) => new Uint8Array(size).map(() => 256 * context.globalThis.Math.random()) ), diff --git a/packages/core/src/workflow/sleep.ts b/packages/core/src/workflow/sleep.ts index 5245745e62..c8848d0c3a 100644 --- a/packages/core/src/workflow/sleep.ts +++ b/packages/core/src/workflow/sleep.ts @@ -15,7 +15,7 @@ export function createSleep(ctx: WorkflowOrchestratorContext) { param: StringValue | Date | number ): Promise { const { promise, resolve } = withResolvers(); - const correlationId = `wait_${ctx.generateCorrelationId('wait')}`; + const correlationId = `wait_${ctx.generateUlid()}`; // Calculate the resume time const resumeAt = parseDurationToDate(param); diff --git a/packages/core/tsconfig.json b/packages/core/tsconfig.json index 7749f99278..98892887a9 100644 --- a/packages/core/tsconfig.json +++ b/packages/core/tsconfig.json @@ -5,5 +5,5 @@ "rootDir": "./src" }, "include": ["src"], - "exclude": ["node_modules", "**/*.test.ts", "src/test-support"] + "exclude": ["node_modules", "**/*.test.ts"] } diff --git a/packages/docs-typecheck/src/type-checker.ts b/packages/docs-typecheck/src/type-checker.ts index ba5ae5d9b0..e4967d8bbb 100644 --- a/packages/docs-typecheck/src/type-checker.ts +++ b/packages/docs-typecheck/src/type-checker.ts @@ -100,6 +100,12 @@ const compilerOptions: ts.CompilerOptions = { '@workflow/serde': [path.join(repoRoot, 'packages/serde/dist/index')], '@workflow/vitest': [path.join(repoRoot, 'packages/vitest/dist/index')], '@workflow/world': [path.join(repoRoot, 'packages/world/dist/index')], + '@workflow/world-sim': [ + path.join(repoRoot, 'packages/world-sim/dist/index'), + ], + '@workflow/world-sim/build': [ + path.join(repoRoot, 'packages/world-sim/dist/build'), + ], // Third-party deps available in docs-typecheck/node_modules zod: [path.join(__dirname, '../node_modules/zod')], ai: [path.join(__dirname, '../node_modules/ai')], diff --git a/packages/world-local/src/fs.test.ts b/packages/world-local/src/fs.test.ts index 100084a80b..1f647b5bb9 100644 --- a/packages/world-local/src/fs.test.ts +++ b/packages/world-local/src/fs.test.ts @@ -976,6 +976,86 @@ describe('fs utilities', () => { } }); }); + + describe('sort-key cursors', () => { + // Slot-numbered events: the file id carries the sort key, and the + // stored `createdAt` deliberately runs backwards relative to it, the + // way a writer that loses a slot race and bumps produces a higher slot + // with an older timestamp. + const RUN_PREFIX = 'run1-'; + const SLOT_COUNT = 12; + const slotId = (slot: number) => `evnt_${String(slot).padStart(26, '0')}`; + + beforeEach(async () => { + const baseTime = new Date('2024-01-01T00:00:00.000Z').getTime(); + const files: Record = {}; + for (let slot = 1; slot <= SLOT_COUNT; slot++) { + const id = slotId(slot); + files[`${RUN_PREFIX}${id}`] = { + id, + name: `event-${slot}`, + createdAt: new Date(baseTime - ms(`${slot}m`)), + }; + } + await createFilesystem(testDir, files); + }); + + const query = (cursor?: string) => + paginatedFileSystemQuery({ + directory: testDir, + schema: TestItemSchema, + filePrefix: RUN_PREFIX, + getCreatedAt: () => null, + getId: (item: TestItem) => item.id, + getSortKey: (item: TestItem) => item.id, + getSortKeyFromFileId: (fileId: string) => + fileId.slice(RUN_PREFIX.length), + sortOrder: 'asc', + limit: 5, + cursor, + }); + + it('pages through the whole log in slot order', async () => { + const seen: string[] = []; + let cursor: string | undefined; + let hasMore = true; + + while (hasMore) { + const page: PaginatedResponse = await query(cursor); + seen.push(...page.data.map((item) => item.id)); + cursor = page.cursor ?? undefined; + hasMore = page.hasMore; + } + + expect(seen).toEqual( + Array.from({ length: SLOT_COUNT }, (_, index) => slotId(index + 1)) + ); + }); + + it('does not read files the cursor has already passed', async () => { + const firstPage = await query(); + assert(firstPage.cursor, 'expected first page cursor to be defined'); + + const readFile = vi.spyOn(fs, 'readFile'); + const secondPage = await query(firstPage.cursor); + const readIds = readFile.mock.calls.map((call) => + path.basename(String(call[0]), '.json') + ); + readFile.mockRestore(); + + // Only the tail past the cursor is opened. Without the filename-level + // prefilter every page reads every file for the run, which makes + // walking a long event log quadratic. + expect(readIds).toEqual( + Array.from( + { length: SLOT_COUNT - firstPage.data.length }, + (_, index) => + `${RUN_PREFIX}${slotId(firstPage.data.length + index + 1)}` + ) + ); + expect(secondPage.data).toHaveLength(5); + }); + }); }); describe('concurrent writes', () => { diff --git a/packages/world-local/src/fs.ts b/packages/world-local/src/fs.ts index 16f740c4f1..29cc2990f8 100644 --- a/packages/world-local/src/fs.ts +++ b/packages/world-local/src/fs.ts @@ -580,24 +580,70 @@ interface PaginatedFileSystemQueryConfig { cursor?: string; getCreatedAt(filename: string): Date | null; getId?(item: T): string; + /** + * Opt an item out of `createdAt` ordering in favor of a total order carried + * by the item itself. + * + * Slot-numbered events are the case this exists for: the slot is assigned + * at the publish, which is the linearization point, while `createdAt` is + * stamped when the request arrives. A writer that loses a slot race and + * bumps therefore lands at a higher slot with an older `createdAt`, and + * ordering by time would hand back a log whose order contradicts the + * positions the World assigned. Return null to keep the `createdAt` + * ordering (ULID-numbered events, and every other entity). + */ + getSortKey?(item: T): string | null; + /** + * The same key as {@link getSortKey}, read off the file id instead of the + * item, so a sort-key cursor can skip files without opening them. + * + * Without it a sort-key scan has no filename-level prefilter and every page + * loads and parses every file for the run, which makes walking a long event + * log quadratic. Return null when the file id does not carry the key; those + * files are kept and decided by the item-level filter. + */ + getSortKeyFromFileId?(fileId: string): string | null; } -// Cursor format: "timestamp|id" for tie-breaking + +// Cursor formats: +// "timestamp|id" β€” createdAt order, id for tie-breaking +// "key:" β€” sort-key order (see getSortKey) +// A run never mixes the two, so a cursor never has to cross formats mid-scan. +export const SORT_KEY_CURSOR_PREFIX = 'key:'; + interface ParsedCursor { timestamp: Date; id: string | null; + sortKey: string | null; } function parseCursor(cursor: string | undefined): ParsedCursor | null { if (!cursor) return null; + if (cursor.startsWith(SORT_KEY_CURSOR_PREFIX)) { + return { + timestamp: new Date(0), + id: null, + sortKey: cursor.slice(SORT_KEY_CURSOR_PREFIX.length), + }; + } + const parts = cursor.split('|'); return { timestamp: new Date(parts[0]), id: parts[1] || null, + sortKey: null, }; } -function createCursor(timestamp: Date, id: string | undefined): string { +function createCursor( + timestamp: Date, + id: string | undefined, + sortKey?: string | null +): string { + if (sortKey) { + return `${SORT_KEY_CURSOR_PREFIX}${sortKey}`; + } return id ? `${timestamp.toISOString()}|${id}` : timestamp.toISOString(); } @@ -616,6 +662,8 @@ export async function paginatedFileSystemQuery( cursor, getCreatedAt, getId, + getSortKey, + getSortKeyFromFileId, } = config; // Validate filePrefix (typically `${runId}-`) so request-derived prefixes @@ -644,7 +692,20 @@ export async function paginatedFileSystemQuery( const parsedCursor = parseCursor(cursor); let candidateFileIds = filteredFileIds; - if (parsedCursor) { + if (parsedCursor?.sortKey && getSortKeyFromFileId) { + // Sort-key cursor: the filename carries the key, so the same strict + // comparison the item-level filter below applies can run here, before any + // file is read. + const cursorSortKey = parsedCursor.sortKey; + candidateFileIds = filteredFileIds.filter((fileId) => { + const key = getSortKeyFromFileId(fileId); + if (key === null) { + return true; + } + const comparison = key.localeCompare(cursorSortKey); + return sortOrder === 'desc' ? comparison < 0 : comparison > 0; + }); + } else if (parsedCursor && !parsedCursor.sortKey) { candidateFileIds = filteredFileIds.filter((fileId) => { const filenameDate = getCreatedAt(`${fileId}.json`); if (filenameDate) { @@ -717,6 +778,23 @@ export async function paginatedFileSystemQuery( for (const item of loadedBatch) { if (!item) continue; + const itemSortKey = getSortKey?.(item) ?? null; + + if (parsedCursor?.sortKey) { + // Sort-key cursor: the key alone is the total order, so there is no + // tie to break. An item without a key cannot be placed relative to + // the cursor at all β€” that would mean a run mixed the two schemes β€” + // so keep it and let the comparator below order it. + if (itemSortKey) { + const comparison = itemSortKey.localeCompare(parsedCursor.sortKey); + if (sortOrder === 'desc' ? comparison >= 0 : comparison <= 0) { + continue; + } + } + validItems.push(item); + continue; + } + // Double-check cursor filtering with actual createdAt from JSON // (in case ULID timestamp differs from stored createdAt) if (parsedCursor) { @@ -746,8 +824,18 @@ export async function paginatedFileSystemQuery( } } - // 5. Sort by createdAt (and by ID for tie-breaking if getId is provided) + // 5. Sort by sortKey when the items carry one, else by createdAt (and by ID + // for tie-breaking if getId is provided) validItems.sort((a, b) => { + if (getSortKey) { + const aKey = getSortKey(a); + const bKey = getSortKey(b); + if (aKey !== null && bKey !== null) { + return sortOrder === 'asc' + ? aKey.localeCompare(bKey) + : bKey.localeCompare(aKey); + } + } const aTime = a.createdAt.getTime(); const bTime = b.createdAt.getTime(); const timeComparison = sortOrder === 'asc' ? aTime - bTime : bTime - aTime; @@ -771,7 +859,8 @@ export async function paginatedFileSystemQuery( items.length > 0 ? createCursor( items[items.length - 1].createdAt, - getId?.(items[items.length - 1]) + getId?.(items[items.length - 1]), + getSortKey?.(items[items.length - 1]) ) : null; diff --git a/packages/world-local/src/index.ts b/packages/world-local/src/index.ts index dfa56584df..b89fd67af5 100644 --- a/packages/world-local/src/index.ts +++ b/packages/world-local/src/index.ts @@ -80,6 +80,10 @@ export function createWorld(args?: Partial): LocalWorld { // events-storage.ts `claimHookResume`), so resumeHook()'s parallel fast // path converges on one event in dev exactly as it does on Vercel. hookResumeDedup: true, + // New runs get dense per-run slot event ids. Runs created before this + // keep their ULIDs; the scheme is pinned by a run's own first event id, + // not by this flag, which only says what new runs get. + slotEventIds: true, }, ...queue, ...storage, diff --git a/packages/world-local/src/storage/events-storage.ts b/packages/world-local/src/storage/events-storage.ts index c6d29d1121..fae85024e8 100644 --- a/packages/world-local/src/storage/events-storage.ts +++ b/packages/world-local/src/storage/events-storage.ts @@ -13,12 +13,14 @@ import { import type { AnyEventRequest, CreateEventParams, + CreateEventRequest, Event, EventResult, Hook, HookCreatedEventRequest, PaginatedResponse, PaginationOptions, + ResolveData, SerializedData, Step, Storage, @@ -28,12 +30,15 @@ import type { import { applyAttributeChanges, EventSchema, + eventIdToSlot, + FIRST_EVENT_SLOT, getMaxEventsPerRun, HookSchema, isChildEntityCreationEvent, isHookEventRequiringExistence, isHookLifecycleEventType, isLegacySpecVersion, + isSlotEventId, isStepEventType, isTerminalRunEventType, isTerminalStepStatus, @@ -41,6 +46,7 @@ import { requiresNewerWorld, SPEC_VERSION_CURRENT, StepSchema, + slotToEventId, ulidToDate, validateAttributeChanges, validateUlidTimestamp, @@ -60,6 +66,8 @@ import { readJSON, readJSONWithFallback, resolveWithinBase, + SORT_KEY_CURSOR_PREFIX, + stripTag, taggedPath, write, writeExclusive, @@ -82,6 +90,7 @@ import { reapPendingHookEvents, releaseHookTokenClaimIfOwnedBy, runTerminalMarkerPath, + scanRunEventIds, withHookTokenClaimLock, } from './helpers.js'; import { @@ -166,6 +175,67 @@ const HookResumeClaimSchema = z.object({ eventId: z.string(), payloadDigest: z.string().optional(), }); + +/** + * Whether `event` is the `hook_received` a resume claim stands for. + * + * The claim names the id its writer INTENDED to publish at, drawn from that + * writer's slot allocator before the append. Under slot ids that intent is not + * a reservation: the allocator is per storage instance, so an instance sharing + * the directory can publish an unrelated event at the same position first, and + * the resume then lands somewhere else. An event read back at the claimed id + * therefore has to be identified, not assumed β€” returning whatever occupies the + * position reports a `run_started` as the resume's own event and silently drops + * the payload. + */ +function isResumeEvent( + event: Event, + claim: z.infer +): boolean { + return ( + event.eventType === 'hook_received' && + event.correlationId === claim.hookId && + // `resumeId` is persisted on every hook_received written through the + // resume path; an event without one predates that and can only be matched + // by position. + (event.resumeId === undefined || event.resumeId === claim.resumeId) + ); +} + +/** + * Finds the event a resume already committed, by the `resumeId` persisted on + * the event itself rather than by the position the claim guessed. + * + * This is the authority the claim's `eventId` only approximates. Reached when + * the claimed position holds nothing (a crash between claim and append) or + * holds an unrelated event (a cross-instance slot collision), so it pays its + * O(run's events) reads on rare paths only. + */ +async function findCommittedResumeEvent( + basedir: string, + runId: string, + claim: z.infer, + tag?: string +): Promise { + const scan = await scanRunEventIds(basedir, runId, tag); + for (const eventId of scan.ids) { + const event = await readJSONWithFallback( + basedir, + 'events', + `${runId}-${eventId}`, + EventSchema, + tag + ); + if ( + event && + event.resumeId === claim.resumeId && + isResumeEvent(event, claim) + ) { + return event; + } + } + return null; +} /** * Whether a token claim held by another `(runId, hookId)` can never become * live again and may therefore be released by a new claimant: @@ -239,6 +309,37 @@ async function readHookRecoveryMarker( * already exists at that exact path β€” which is the correct * "already-published" semantic. */ +/** + * Log order for a slot-numbered run is slot order, not `createdAt` order. A + * writer stamps `createdAt` when it enters `create()` but only claims its slot + * at publish time, so a writer that loses a slot race and bumps ends up with a + * higher slot and an older timestamp than the writer that beat it. Slot order + * is the one both writers agree on, and it is what makes the log dense and + * position-addressable, so it wins. + * + * Returns `null` for a ULID-numbered run, which falls back to + * `(createdAt, eventId)` β€” the two never mix within one run. + */ +function eventSortKey(event: Event): string | null { + return isSlotEventId(event.eventId) ? event.eventId : null; +} + +/** + * The same key as {@link eventSortKey}, recovered from an event file's name. + * + * Event files are named `${runId}-${eventId}` plus an optional tag suffix, so + * a run-scoped listing can read the slot without opening the file. That lets a + * sort-key cursor discard the pages it has already returned on the filename + * alone; without it every page of a long log loads and parses every event file + * for the run. + * + * Returns `null` for a ULID-numbered event, which has no slot to compare. + */ +function eventSortKeyFromFileId(runId: string, fileId: string): string | null { + const eventId = stripTag(fileId).slice(runId.length + 1); + return isSlotEventId(eventId) ? eventId : null; +} + async function findExistingHookCreatedEventId( basedir: string, runId: string, @@ -506,6 +607,159 @@ export function createEventsStorage( cachedEventBytes.clear(); cachedPathsByRunId.clear(); totalCachedEventBytes = 0; + runSlotState.clear(); + } + + // ------------------------------------------------------------------ + // Slot allocation + // ------------------------------------------------------------------ + // + // Event ids are per-run positions (`evnt_` + a 26-char zero-padded + // decimal), dense and 1-based, so the count of a run's events and the + // highest id are the same number. That equivalence is what lets a writer + // state its position with a single integer, and it only holds if the World + // never leaves a hole: a slot is claimed by the publish that occupies it, + // never reserved ahead of a write that might still be rejected. + // + // Runs created before slot ids keep their ULIDs for life. A log may not mix + // the two schemes (`events.list` sorts on the id, and they do not + // interleave), so the ids already on disk are the authoritative pin β€” no + // spec-version negotiation is involved. `null` state below means "this run + // is ULID-numbered". + // + // The map holds the highest slot this instance has seen PUBLISHED for a + // run, never a reservation. A draw reports `published + 1` and leaves the + // entry alone, so a write rejected anywhere between its draw and its + // publish costs nothing: the next writer draws the same position. That is + // what keeps the log dense, and it is not a rare path β€” a duplicate + // `step_started` from a concurrent replay is rejected on every storm. + // + // The cost is that two in-flight writers hold the same candidate. The + // exclusive publish arbitrates and the loser bumps, which is the same + // mechanism two instances sharing one directory (a test-only configuration + // this backend supports) already rely on. + const runSlotState = new Map(); + + function slotStateKey(runId: string): string { + return tag ? `${runId}.${tag}` : runId; + } + + /** Whether a slot is occupied by a reader-visible event file. */ + async function slotOccupied(runId: string, slot: number): Promise { + const fileId = `${runId}-${slotToEventId(slot)}`; + for (const candidate of tag + ? [ + taggedPath(basedir, 'events', fileId, tag), + taggedPath(basedir, 'events', fileId), + ] + : [taggedPath(basedir, 'events', fileId)]) { + try { + await fs.stat(candidate); + return true; + } catch (error) { + if ((error as NodeJS.ErrnoException).code !== 'ENOENT') { + throw error; + } + } + } + return false; + } + + /** + * Draws the candidate slot for `runId`, or null when the run is + * ULID-numbered and should keep minting ULIDs. + * + * `atLeast` re-floors the candidate after a lost publish. The directory scan + * runs once per run per instance (and again on a `rescan`), not once per + * write. + * + * The watermark alone is a lower bound: it only counts publishes this + * instance made or last scanned for, so another instance sharing the + * directory can be ahead of it. The candidate is therefore probed upward + * until it lands on a free position β€” one `stat` that returns ENOENT in the + * uncontended case. Skipping the probe would be tolerable for an ordinary + * write (the exclusive publish bumps it), but not for the writes that + * record their candidate in a durable claim for other writers to converge + * on: a claim naming an occupied slot never converges. + */ + async function drawEventSlot( + runId: string, + opts?: { rescan?: boolean; atLeast?: number } + ): Promise { + const key = slotStateKey(runId); + let state = runSlotState.get(key); + if (state === undefined || opts?.rescan) { + const scan = await scanRunEventIds(basedir, runId, tag); + if (state === undefined) { + // A run with no events yet is brand new: it starts on slots. A run + // whose visible events are ULIDs stays on ULIDs for life. + state = + scan.count > 0 && !scan.usesSlots + ? null + : { published: scan.maxSlot }; + runSlotState.set(key, state); + } else if (state !== null) { + state.published = Math.max(state.published, scan.maxSlot); + } + } + if (state === null) { + return null; + } + let slot = Math.max( + state.published + 1, + FIRST_EVENT_SLOT, + opts?.atLeast ?? FIRST_EVENT_SLOT + ); + while (await slotOccupied(runId, slot)) { + state.published = Math.max(state.published, slot); + slot += 1; + } + return slot; + } + + /** + * Records that `eventId` is now on disk, so the next draw starts above it. + * + * Only a committed publish moves the watermark. A draw that never published + * leaves no trace, which is the whole reason the log has no holes. + * + * A no-op for ULID-numbered runs, where ids are not positions, and for runs + * this instance has never drawn for β€” the first draw scans the directory. + */ + function notePublishedSlot(runId: string, eventId: string): void { + const slot = eventIdToSlot(eventId); + if (slot === null) { + return; + } + const state = runSlotState.get(slotStateKey(runId)); + if (state) { + state.published = Math.max(state.published, slot); + } + } + + /** Mints the next event id for `runId` under whichever scheme it uses. */ + async function mintEventId(runId: string): Promise { + const slot = await drawEventSlot(runId); + return slot === null ? `evnt_${monotonicUlid()}` : slotToEventId(slot); + } + + /** + * Mints the key a terminal transition appends under, re-derived at its + * linearization point (after the marker + reap) so it sorts after any + * `hook_received` that legitimately won the promote arbitration. + * + * For slot runs the rescan is the whole mechanism: it floors the candidate + * past anything another instance promoted while this invocation was + * stalled, and the drawn slot dominates by construction. + */ + async function mintDominantEventKey( + runId: string + ): Promise<{ eventId: string; createdAt: Date }> { + const slot = await drawEventSlot(runId, { rescan: true }); + if (slot !== null) { + return { eventId: slotToEventId(slot), createdAt: new Date() }; + } + return mintRunDominantEventKey(basedir, runId, tag); } function cacheEvent( @@ -566,16 +820,47 @@ export function createEventsStorage( } } - async function storeEvent(event: Event): Promise { - const eventPath = taggedPath( - basedir, - 'events', - `${event.runId}-${event.eventId}`, - tag - ); - const serializedEvent = JSON.stringify(event, jsonReplacer, 2); - await write(eventPath, serializedEvent); - rememberStoredEvent(event, eventPath, serializedEvent); + /** + * Publishes a synthetic event (one this call writes in addition to the + * event it was asked for) and returns it under the id it actually landed + * on. + * + * A slot is a position, so the candidate this event was drawn at may have + * been taken by a concurrent writer between the draw and here; the + * exclusive create detects that and the next position is tried. ULID ids + * are globally unique, so a collision there can only be a retry of this + * same event and the write stays an overwrite. + */ + async function storeEvent(event: Event): Promise { + let current = event; + for (let attempt = 0; ; attempt++) { + const eventPath = taggedPath( + basedir, + 'events', + `${current.runId}-${current.eventId}`, + tag + ); + const serializedEvent = JSON.stringify(current, jsonReplacer, 2); + const slot = eventIdToSlot(current.eventId); + if (slot === null) { + await write(eventPath, serializedEvent); + rememberStoredEvent(current, eventPath, serializedEvent); + return current; + } + if (await writeExclusive(eventPath, serializedEvent)) { + notePublishedSlot(current.runId, current.eventId); + rememberStoredEvent(current, eventPath, serializedEvent); + return current; + } + const next = await drawEventSlot(current.runId, { + rescan: attempt > 0 && attempt % 8 === 0, + atLeast: slot + 1, + }); + // `next` is null only for a ULID-numbered run, which the branch above + // already returned for. + assert(next !== null); + current = { ...current, eventId: slotToEventId(next) }; + } } const queryRunEvents = (runId: string, pagination: PaginationOptions) => @@ -589,6 +874,8 @@ export function createEventsStorage( cursor: pagination.cursor, getCreatedAt: getObjectCreatedAt('evnt'), getId: (event) => event.eventId, + getSortKey: eventSortKey, + getSortKeyFromFileId: (fileId) => eventSortKeyFromFileId(runId, fileId), }); // Per-instance in-process mutexes. Two storage instances sharing @@ -610,7 +897,7 @@ export function createEventsStorage( const stepLocks = new Map>(); const hookLocks = new Map>(); - return { + const storage: LocalEventsStorage = { clearCache, async create( runId: string | null, @@ -689,12 +976,25 @@ export function createEventsStorage( return createImpl(); async function createImpl(): Promise { - // Most paths use the freshly-generated candidate eventId. The + // Most paths use the freshly-drawn candidate eventId. The // hook_created dedup-recovery path below may reassign it to // the canonical eventId persisted in the durable token claim // so concurrent / cross-process workers converge on a single - // event in the log. - let eventId = `evnt_${monotonicUlid()}`; + // event in the log; `eventIdPinned` records that, because a pinned + // id must never be bumped past a slot collision (bumping would + // defeat the convergence and duplicate the event). + // + // Drawn below rather than here: slots are positions, so they must be + // drawn in publish order. The resilient-start path writes a synthetic + // `run_created` that has to precede this event in the log, and a slot + // drawn at function entry would sort after it. + let eventId = ''; + let eventIdPinned = false; + // The eventId currently recorded in this resume's `(runId, resumeId)` + // claim, when one was written or read below. An unpinned publish is + // free to land somewhere else, and the claim is the fast path other + // writers read first, so it is corrected once the append commits. + let resumeClaimRecordedId: string | null = null; const now = new Date(); // For run_created events, use client-provided runId or generate one server-side @@ -803,7 +1103,9 @@ export function createEventsStorage( if (created) { // We created the run β€” also write the run_created event. - const runCreatedEventId = `evnt_${monotonicUlid()}`; + // Drawn before this invocation's own id so it takes the + // earlier slot: it must replay first. + const runCreatedEventId = await mintEventId(effectiveRunId); const runCreatedEvent: Event = { eventType: 'run_created', runId: effectiveRunId, @@ -838,6 +1140,18 @@ export function createEventsStorage( } } + // Draw this event's id now that any synthetic `run_created` above has + // taken the earlier slot. A slot draw is a candidate, not a + // reservation: the many rejections below (a duplicate step_started + // from a concurrent replay, a terminal run, a step already in a + // terminal state) return without publishing, and the position stays + // available to the next writer. + eventId = await mintEventId(effectiveRunId); + + // These events are rejected on a non-existent run to match the + // postgres and vercel worlds, which both surface this as a + // WorkflowRunNotFoundError rather than silently persisting an + // event for a run that was never created. if ( !currentRun && (data.eventType === 'run_failed' || @@ -895,18 +1209,17 @@ export function createEventsStorage( currentRun.status === 'cancelled' ) { // Return existing state (idempotent) - const event: Event = { + const stored = await storeEvent({ ...data, runId: effectiveRunId, eventId, createdAt: now, specVersion: effectiveSpecVersion, - }; - await storeEvent(event); + }); const resolveData = params?.resolveData ?? DEFAULT_RESOLVE_DATA_OPTION; return { - event: stripEventDataRefs(event, resolveData), + event: stripEventDataRefs(stored, resolveData), run: currentRun, ...(currentRun ? { maxEvents: getMaxEventsPerRun() } : {}), }; @@ -1033,13 +1346,22 @@ export function createEventsStorage( !committedClaim.payloadDigest || committedClaim.payloadDigest === params.resumePayloadDigest) ) { - const committedEvent = await readJSONWithFallback( + const atClaimedId = await readJSONWithFallback( basedir, 'events', `${effectiveRunId}-${committedClaim.eventId}`, EventSchema, tag ); + const committedEvent = + atClaimedId && isResumeEvent(atClaimedId, committedClaim) + ? atClaimedId + : await findCommittedResumeEvent( + basedir, + effectiveRunId, + committedClaim, + tag + ); if (committedEvent) { return { event: committedEvent }; } @@ -1115,20 +1437,42 @@ export function createEventsStorage( `hook_received resumeId "${params.resumeId}" already recorded with a different payload` ); } - const existing = await readJSONWithFallback( + const atClaimedId = await readJSONWithFallback( basedir, 'events', `${effectiveRunId}-${claim.eventId}`, EventSchema, tag ); - if (existing) { - return { event: existing }; + if (atClaimedId && isResumeEvent(atClaimedId, claim)) { + return { event: atClaimedId }; + } + // Either nothing is at the claimed position, or something that + // is not this resume is. The claim's `eventId` is only where its + // writer meant to append, so before concluding the resume is + // uncommitted, look for it by the `resumeId` persisted on the + // event. + const committed = await findCommittedResumeEvent( + basedir, + effectiveRunId, + claim, + tag + ); + if (committed) { + return { event: committed }; + } + // The resume really is uncommitted: a crash between the claim + // write and the append. Take over the append. Adopt the claimed + // position when it is still free β€” under ULIDs it always is, and + // adopting keeps two takers writing the same path so one loses + // the exclusive create instead of publishing a second event. + // When an unrelated event holds it, there is nothing to converge + // on: keep this writer's own id and let the publish bump. + resumeClaimRecordedId = claim.eventId; + if (!atClaimedId) { + eventId = claim.eventId; + eventIdPinned = true; } - // Claim exists but its event is not yet visible (a crash between - // the claim write and the append). Adopt the pinned eventId and - // fall through to (re)write the event idempotently at that path. - eventId = claim.eventId; return null; }; @@ -1142,9 +1486,23 @@ export function createEventsStorage( return converged; } } else { - // Reserve the claim (pinning this candidate eventId) before the + // Reserve the claim (naming this candidate eventId) before the // append. If a concurrent/cross-process writer reserved it first, - // converge on their pinned event instead. + // converge on their event instead. + // + // Under ULIDs the candidate is pinned: the id is globally unique, + // so the only writer that can collide with it is the other writer + // of this same resume, and both must land on the one event. + // + // Under slot ids it cannot be. A slot is a position, not a name: + // another instance's allocator hands out the same number for a + // different event, and refusing to bump would fail this resume's + // append outright. So the claimed id is a hint, the publish is + // free to move, and `converge` identifies the resume's event by + // its persisted `resumeId`. The claim is rewritten with the id + // actually published once the append commits. + eventIdPinned = !isSlotEventId(eventId); + resumeClaimRecordedId = eventId; const won = await writeExclusive( claimPath, JSON.stringify({ @@ -1158,6 +1516,9 @@ export function createEventsStorage( } satisfies z.infer) ); if (!won) { + // Someone else's claim is the durable one now; this writer's + // candidate is not what the claim records. + resumeClaimRecordedId = null; const winner = await readJSON(claimPath, HookResumeClaimSchema); if (winner) { const converged = await converge(winner); @@ -1271,11 +1632,7 @@ export function createEventsStorage( // strictly dominates all visible events of the run guarantees the // terminal event replays last. See mintRunDominantEventKey for // the dominance argument. - const dominantKey = await mintRunDominantEventKey( - basedir, - effectiveRunId, - tag - ); + const dominantKey = await mintDominantEventKey(effectiveRunId); eventId = dominantKey.eventId; event = { ...event, eventId, createdAt: dominantKey.createdAt }; } @@ -1662,13 +2019,17 @@ export function createEventsStorage( ); // Write the synthetic step_created event so replay observes it // (the client step consumer sets hasCreatedEvent only on a - // step_created event). Its eventId is a fresh monotonic ULID. - // Ordering vs. the step_started event row does not affect - // correctness: the step_started consumer is a no-op and only - // step_created flips hasCreatedEvent, so the end state is the - // same whichever sorts first β€” this matches the resilient - // run_started β†’ run_created precedent in this file. - const stepCreatedEventId = `evnt_${monotonicUlid()}`; + // step_created event). Its id comes from the run's own + // allocator: minting a ULID here would put a second identity + // scheme in a slot-numbered log, and `events.list` cannot + // paginate a mixed log (a ULID id has no sort key, so it lands + // on every page and the cursor eventually repeats). + // + // This publishes into the position the step_started event is + // still only a candidate for, so the synthetic step_created + // sorts ahead of the step_started that triggered it and the + // step_started bumps up one. + const stepCreatedEventId = await mintEventId(effectiveRunId); const stepCreatedEvent: Event = { eventType: 'step_created', runId: effectiveRunId, @@ -1681,15 +2042,7 @@ export function createEventsStorage( input: lazyData.input, }, }; - await writeJSON( - taggedPath( - basedir, - 'events', - `${effectiveRunId}-${stepCreatedEventId}`, - tag - ), - stepCreatedEvent - ); + await storeEvent(stepCreatedEvent); validatedStep = createdStep; stepCreatedLazily = true; } @@ -1983,8 +2336,12 @@ export function createEventsStorage( canonicalEventId = pinned; } - // The canonical ULID also makes converging writes byte-identical. + // Pinned: this id is the convergence point for every writer of + // this hook, so it must not be bumped past a slot collision. A + // collision here means the canonical event is already published, + // which is exactly the duplicate the handler below repairs from. eventId = canonicalEventId; + eventIdPinned = true; const canonicalCreatedAt = ulidToDate(eventId.replace(/^evnt_/, '')) ?? now; event = { @@ -2017,11 +2374,11 @@ export function createEventsStorage( specVersion: effectiveSpecVersion, }; - await storeEvent(conflictEvent); + const storedConflict = await storeEvent(conflictEvent); const resolveData = params?.resolveData ?? DEFAULT_RESOLVE_DATA_OPTION; return { - event: stripEventDataRefs(conflictEvent, resolveData), + event: stripEventDataRefs(storedConflict, resolveData), run, step, hook: undefined, @@ -2253,12 +2610,67 @@ export function createEventsStorage( throw new HookNotFoundError(data.correlationId); } - const compositeKey = `${effectiveRunId}-${eventId}`; - const eventPath = taggedPath(basedir, 'events', compositeKey, tag); + let eventPath = taggedPath( + basedir, + 'events', + `${effectiveRunId}-${eventId}`, + tag + ); // Capture the serialized payload before the write's `await` so the // cached snapshot can't observe a later mutation (see // rememberStoredEvent). - const serializedEvent = JSON.stringify(event, jsonReplacer, 2); + let serializedEvent = JSON.stringify(event, jsonReplacer, 2); + + /** + * Moves this event to the next free slot after a lost publish, and + * reports whether it could. + * + * A slot id is a position in the run's log, not a globally unique + * token, so losing the publish means another writer took the + * position β€” an ordinary concurrent write. The World's contract is to + * bump and commit rather than reject: `create` must not fail for a + * reason its caller could not have avoided. Bumping is refused for: + * + * - ULID-numbered runs, where ids ARE globally unique and a collision + * really is a duplicate publish that must surface; + * - ids pinned by a durable claim (`hook_created`'s canonical id, + * `hook_received`'s resume claim), which exist precisely so two + * writers converge on ONE event β€” bumping would publish a second. + * + * A pinned id is only ever read back from a claim its own writer + * recorded before publishing, and that writer bumps only when the + * position it recorded is already occupied. So an adopter that finds + * the claim stale finds the slot taken too: it collides, and the + * collision is the benign-duplicate path this dedup already has. It + * never publishes a second `hook_created` into a free slot. + */ + const bumpEventSlot = async (attempt: number): Promise => { + const current = eventIdToSlot(eventId); + if (eventIdPinned || current === null) { + return false; + } + // Every failure moves this write up by at least one position, so + // this terminates even under heavy contention. Rescan periodically + // so a batch committed by another instance is skipped in one step + // rather than one slot at a time. + const slot = await drawEventSlot(effectiveRunId, { + rescan: attempt > 0 && attempt % 8 === 0, + atLeast: current + 1, + }); + if (slot === null) { + return false; + } + eventId = slotToEventId(slot); + event = { ...event, eventId }; + eventPath = taggedPath( + basedir, + 'events', + `${effectiveRunId}-${eventId}`, + tag + ); + serializedEvent = JSON.stringify(event, jsonReplacer, 2); + return true; + }; // Cross-process terminal-run guard for `hook_received`. A terminal // transition (run_completed / run_failed / run_cancelled) in ANY @@ -2292,73 +2704,90 @@ export function createEventsStorage( // reap has passed necessarily stages after the marker was // committed, so step 3 rejects it. Rejections before step 4 unlink // a file no reader can see. - let eventPublished: boolean; - if (data.eventType === 'hook_received') { - // Step 1: fast path. The marker is the authoritative durable - // signal; the run-state read additionally rejects runs whose - // terminal state was written without a marker (e.g. runs that - // terminated on an older storage version). - const terminalByMarker = await isRunTerminalCommitted( - basedir, - effectiveRunId, - tag - ); - const runNow = terminalByMarker - ? null - : await readJSONWithFallback( - basedir, - 'runs', - effectiveRunId, - WorkflowRunSchema, - tag - ); - if ( - terminalByMarker || - (runNow && isTerminalWorkflowRunStatus(runNow.status)) - ) { - throw new RunExpiredError( - `Workflow run "${effectiveRunId}" is already in a terminal state` - ); - } - - const stagedPath = pendingHookEventPath( - basedir, - effectiveRunId, - eventId, - tag - ); - const staged = await writeExclusive(stagedPath, serializedEvent); - if (!staged) { - // eventId is a freshly generated ULID; its staging path can - // only be occupied by a previous crashed attempt of this very - // event, which never promoted. Surface the same conflict shape - // as a visible-path collision. - throw new EntityConflictError( - `Event "${eventId}" already exists for run "${effectiveRunId}"` + let eventPublished = false; + for (let attempt = 0; ; attempt++) { + if (data.eventType === 'hook_received') { + // Step 1: fast path. The marker is the authoritative durable + // signal; the run-state read additionally rejects runs whose + // terminal state was written without a marker (e.g. runs that + // terminated on an older storage version). + const terminalByMarker = await isRunTerminalCommitted( + basedir, + effectiveRunId, + tag ); - } - try { - if (await isRunTerminalCommitted(basedir, effectiveRunId, tag)) { + const runNow = terminalByMarker + ? null + : await readJSONWithFallback( + basedir, + 'runs', + effectiveRunId, + WorkflowRunSchema, + tag + ); + if ( + terminalByMarker || + (runNow && isTerminalWorkflowRunStatus(runNow.status)) + ) { throw new RunExpiredError( `Workflow run "${effectiveRunId}" is already in a terminal state` ); } - const promoted = await promoteExclusive(stagedPath, eventPath); - if (promoted === 'missing') { - // A terminal transition reaped the staged file between the - // check and the link β€” the atomic loss of the arbitration. - throw new RunExpiredError( - `Workflow run "${effectiveRunId}" is already in a terminal state` + + const stagedPath = pendingHookEventPath( + basedir, + effectiveRunId, + eventId, + tag + ); + const staged = await writeExclusive(stagedPath, serializedEvent); + if (!staged) { + // The staging path can be occupied by a previous crashed + // attempt of this very event (which never promoted), or, under + // slot ids, by a concurrent writer holding the same position. + // Both are handled the same way: fall through to the bump + // below, which moves off the position when it can and surfaces + // the conflict when it cannot. + if (await bumpEventSlot(attempt)) { + continue; + } + throw new EntityConflictError( + `Event "${eventId}" already exists for run "${effectiveRunId}"` ); } - eventPublished = promoted === 'linked'; - } finally { - // The staged path is not reader-visible; removing it is pure - // cleanup on every outcome (already gone when reaped). - await deleteJSON(stagedPath).catch(() => {}); + try { + if (await isRunTerminalCommitted(basedir, effectiveRunId, tag)) { + throw new RunExpiredError( + `Workflow run "${effectiveRunId}" is already in a terminal state` + ); + } + const promoted = await promoteExclusive(stagedPath, eventPath); + if (promoted === 'missing') { + // A terminal transition reaped the staged file between the + // check and the link β€” the atomic loss of the arbitration. + throw new RunExpiredError( + `Workflow run "${effectiveRunId}" is already in a terminal state` + ); + } + eventPublished = promoted === 'linked'; + } finally { + // The staged path is not reader-visible; removing it is pure + // cleanup on every outcome (already gone when reaped). + await deleteJSON(stagedPath).catch(() => {}); + } + } else { + eventPublished = await writeExclusive(eventPath, serializedEvent); + } + + if (eventPublished) { + // The position is occupied now, so the next draw for this run + // starts above it. Only a committed publish moves the watermark. + notePublishedSlot(effectiveRunId, eventId); + break; + } + if (!(await bumpEventSlot(attempt))) { + break; } - } else { - eventPublished = await writeExclusive(eventPath, serializedEvent); } if (!eventPublished) { @@ -2391,6 +2820,34 @@ export function createEventsStorage( // replay can serve it without rereading from disk. rememberStoredEvent(event, eventPath, serializedEvent); + // Point the resume claim at where the event actually landed. An + // unpinned publish bumps past occupied slots, so the id the claim + // recorded before the append can be stale; leaving it stale would + // send every later reader of this resume down the `resumeId` scan + // instead of the single read the claim exists to provide. Plain + // overwrite, not exclusive-create: the claim is already this + // writer's, and only the eventId changes. + if ( + data.eventType === 'hook_received' && + params?.resumeId && + resumeClaimRecordedId !== null && + resumeClaimRecordedId !== eventId + ) { + await write( + hookResumeClaimPath(basedir, effectiveRunId, params.resumeId), + JSON.stringify({ + runId: effectiveRunId, + resumeId: params.resumeId, + hookId: data.correlationId, + eventId, + ...(params.resumePayloadDigest + ? { payloadDigest: params.resumePayloadDigest } + : {}), + } satisfies z.infer), + { overwrite: true } + ); + } + // Write the hook entity ONLY now that the event publish has // committed. Doing this earlier (in the `hook_created` // branch above) would mutate an already-committed hook @@ -2551,6 +3008,9 @@ export function createEventsStorage( cursor: params.pagination?.cursor, getCreatedAt: getObjectCreatedAt('evnt'), getId: (event) => event.eventId, + getSortKey: eventSortKey, + getSortKeyFromFileId: (fileId) => + eventSortKeyFromFileId(params.runId, fileId), }); // If resolveData is "none", remove eventData from events @@ -2566,4 +3026,88 @@ export function createEventsStorage( return result; }, }; + + /** + * The report half of bump-and-report: the events sitting on the slots + * between the one the writer asked for and the one its write landed on. + * + * Wrapped around `create` rather than folded into it because `create` has a + * dozen commit points (dedup recovery, hook conflict, lazy step creation) + * and the report is the same at every one of them: read the committed id, + * read back what is below it. + * + * The read is a directory scan, so it only runs when the write actually + * skipped a slot. `hasMore` says the set is a lower bound: another instance + * may hold a lower slot it has not published yet, and a draw whose publish + * was lost leaves one permanently empty. + */ + async function reportSkippedSlots( + result: EventResult, + askedFor: number, + resolveData: ResolveData + ): Promise { + if (!result.event) { + return result; + } + const committedSlot = eventIdToSlot(result.event.eventId); + if ( + committedSlot === null || + askedFor < FIRST_EVENT_SLOT || + committedSlot <= askedFor + 1 + ) { + return result; + } + const span = committedSlot - askedFor - 1; + const page = await storage.list({ + runId: result.event.runId, + pagination: { + cursor: `${SORT_KEY_CURSOR_PREFIX}${slotToEventId(askedFor)}`, + limit: span, + sortOrder: 'asc', + }, + resolveData, + }); + // The cursor is exclusive and the page is in slot order, so a dense log + // yields exactly the skipped slots. A hole lets the page reach past the + // committed slot, which is this writer's own event and anything a later + // writer already published: neither is something it skipped over. + const committedEventId = result.event.eventId; + const events = page.data.filter( + (event) => event.eventId < committedEventId + ); + return { + ...result, + events, + // Deliberately no cursor: the report is a lower bound on what this write + // skipped over, not a page the caller has now read to the end of, so it + // must not advance the caller's read position. + cursor: null, + hasMore: events.length < committedSlot - askedFor - 1, + }; + } + + const create = (async ( + runId: string, + data: CreateEventRequest, + params?: CreateEventParams + ): Promise => { + if (params?.eventCount === undefined) { + return storage.create(runId, data, params); + } + const resolveData = params.resolveData ?? DEFAULT_RESOLVE_DATA_OPTION; + const result = await storage.create(runId, data, params); + // `sinceCursor` and the skipped-slot report share `events`/`cursor`/ + // `hasMore`, and the runtime sends both on the same write. The delta wins: + // the skipped slots all sit above the cursor, so it is a strict superset, + // and it is the only one of the two that advances `cursor`. Narrowing + // `events` to the report while leaving the delta's cursor would tell the + // caller it has read a range it was only handed part of, and the rest + // would never be fetched again. + if (typeof params.sinceCursor === 'string') { + return result; + } + return reportSkippedSlots(result, params.eventCount, resolveData); + }) as LocalEventsStorage['create']; + + return { ...storage, create }; } diff --git a/packages/world-local/src/storage/helpers.ts b/packages/world-local/src/storage/helpers.ts index 605e8b3370..9d599dc670 100644 --- a/packages/world-local/src/storage/helpers.ts +++ b/packages/world-local/src/storage/helpers.ts @@ -2,6 +2,7 @@ import { createHash } from 'node:crypto'; import fs from 'node:fs/promises'; import path from 'node:path'; import { WorkflowWorldError } from '@workflow/errors'; +import { eventIdToSlot } from '@workflow/world'; import { lock } from 'proper-lockfile'; import { decodeTime, monotonicFactory } from 'ulid'; import { z } from 'zod'; @@ -234,27 +235,86 @@ export async function reapPendingHookEvents( * >= every visible event's `createdAt`, which was stamped at that event's * `createImpl()` entry β€” before its publish, and thus before this call. * Equal-`createdAt` ties fall to the strictly-dominant eventId. + * + * ULID-numbered runs only. A slot-numbered run needs no temporal argument: + * the next slot dominates every allocated one by construction, so its + * terminal transition just draws from the run's slot allocator after the + * reap. Callers pick the branch (see `mintDominantEventKey` in + * events-storage.ts). */ export async function mintRunDominantEventKey( basedir: string, runId: string, tag?: string ): Promise<{ eventId: string; createdAt: Date }> { + const scan = await scanRunEventIds(basedir, runId, tag); + + let ts = Date.now(); + if (scan.maxId) { + try { + const maxTs = decodeTime(scan.maxId.replace(/^evnt_/, '')); + if (ts <= maxTs) { + ts = maxTs + 1; + } + } catch { + // Malformed eventId in the log β€” fall back to the wall clock. + } + } + return { eventId: `evnt_${monotonicUlid(ts)}`, createdAt: new Date(ts) }; +} + +/** + * What a run's already-published event ids say about its identity scheme. + * + * A run keeps the scheme it was created under for its whole life (a log may + * not mix ULID and slot ids β€” `events.list` sorts on the id, and the two + * schemes do not interleave), so the ids on disk are the authoritative pin. + * `usesSlots` is false for a run with no events yet; the caller decides what a + * brand-new run gets. + */ +export interface RunEventIdScan { + /** Highest reader-visible event id, or null when the run has no events. */ + maxId: string | null; + /** Whether the run's ids are slot-numbered. */ + usesSlots: boolean; + /** Highest allocated slot, or 0 when the run has none. */ + maxSlot: number; + /** Number of reader-visible events found for the run. */ + count: number; + /** Reader-visible event ids, tag stripped, in directory order. */ + ids: string[]; +} + +/** + * Scans the events directory for one run's ids, honoring tag visibility. + * + * O(all event files), like every other directory-walking read in this + * backend. Callers that run it per write memoize the result and use the + * publish itself to detect when the memo has fallen behind. + */ +export async function scanRunEventIds( + basedir: string, + runId: string, + tag?: string +): Promise { let files: string[] = []; try { files = await fs.readdir(path.join(basedir, 'events')); } catch (error) { // Only ENOENT ("no events directory yet") means there is provably - // nothing visible to dominate. Any other failure would silently mint a - // wall-clock key with no dominance guarantee over an already-accepted - // hook β€” abort the terminal transition instead; its retry re-runs this - // scan. + // nothing visible. Any other failure would silently report an empty run, + // which would mint a colliding slot / a non-dominant ULID β€” let the + // caller's retry re-run the scan instead. if ((error as NodeJS.ErrnoException).code !== 'ENOENT') { throw error; } } const prefix = `${runId}-`; - let maxUlid: string | null = null; + const ids: string[] = []; + let maxId: string | null = null; + let maxSlot = 0; + let usesSlots = false; + let count = 0; for (const file of files) { if (!file.startsWith(prefix) || !file.endsWith('.json')) { continue; @@ -266,22 +326,18 @@ export async function mintRunDominantEventKey( continue; } const candidate = stripTag(fileId).slice(prefix.length); - if (!maxUlid || candidate > maxUlid) { - maxUlid = candidate; + count += 1; + ids.push(candidate); + if (!maxId || candidate > maxId) { + maxId = candidate; } - } - let ts = Date.now(); - if (maxUlid) { - try { - const maxTs = decodeTime(maxUlid.replace(/^evnt_/, '')); - if (ts <= maxTs) { - ts = maxTs + 1; - } - } catch { - // Malformed eventId in the log β€” fall back to the wall clock. + const slot = eventIdToSlot(candidate); + if (slot !== null) { + usesSlots = true; + if (slot > maxSlot) maxSlot = slot; } } - return { eventId: `evnt_${monotonicUlid(ts)}`, createdAt: new Date(ts) }; + return { maxId, usesSlots, maxSlot, count, ids }; } /** diff --git a/packages/world-local/src/storage/hook-resume-dedup.test.ts b/packages/world-local/src/storage/hook-resume-dedup.test.ts index 48f545033b..1159330c8f 100644 --- a/packages/world-local/src/storage/hook-resume-dedup.test.ts +++ b/packages/world-local/src/storage/hook-resume-dedup.test.ts @@ -4,6 +4,7 @@ import path from 'node:path'; import { SPEC_VERSION_CURRENT, type Storage } from '@workflow/world'; import { afterEach, beforeEach, describe, expect, it } from 'vitest'; import { createHook, createRun, disposeHook } from '../test-helpers.js'; +import { hookResumeClaimPath } from './helpers.js'; import { createStorage } from './index.js'; // When a run carries the `hookResumeInputVersion` marker, the parallel resume @@ -148,6 +149,120 @@ describe('world-local hook_received resume dedup', () => { ).rejects.toThrow(); }); + // A slot allocator is per storage instance, and two instances share the + // directory whenever a dev server serves the hook request from one module + // instance and runs the queue in another. The resume claim names the id its + // writer MEANT to append at, drawn before the append, so a second instance + // can publish an unrelated event at that position first. + describe('when another instance takes the position the claim named', () => { + async function seedStaleAllocator(runId: string) { + // `other` scans the log once, then counts forward in memory. Writing + // through `storage` afterwards fills the positions `other` still thinks + // are free. + const other = createStorage(testDir); + const attr = (from: Storage, key: string) => + from.events.create(runId, { + eventType: 'attr_set', + specVersion: SPEC_VERSION_CURRENT, + correlationId: `attr_${key}`, + eventData: { + changes: [{ key, value: 'x' }], + writer: { type: 'workflow' }, + }, + }); + await attr(other, 'seed'); + await attr(storage, 'ahead_1'); + await attr(storage, 'ahead_2'); + return other; + } + + it('still commits the resume, at the position actually free', async () => { + const { runId, hook } = await setup(); + const other = await seedStaleAllocator(runId); + const payload = new Uint8Array([1, 2, 3]); + + const result = await other.events.create( + runId, + { + eventType: 'hook_received', + specVersion: SPEC_VERSION_CURRENT, + correlationId: hook.hookId, + eventData: { token: hook.token, payload }, + }, + { resumeId: 'resume_1', resumePayloadDigest: 'digest_1' } + ); + + // The reported event must be this resume's own. Returning whatever sits + // at the claimed position reports an `attr_set` as the resume's event + // and drops the payload without an error. + expect(result.event.eventType).toBe('hook_received'); + expect(result.event.resumeId).toBe('resume_1'); + expect(await countHookReceived(runId)).toBe(1); + }); + + it('converges the redelivery on the committed event, not on the occupant', async () => { + const { runId, hook } = await setup(); + const other = await seedStaleAllocator(runId); + const payload = new Uint8Array([1, 2, 3]); + + const first = await other.events.create( + runId, + { + eventType: 'hook_received', + specVersion: SPEC_VERSION_CURRENT, + correlationId: hook.hookId, + eventData: { token: hook.token, payload }, + }, + { resumeId: 'resume_1', resumePayloadDigest: 'digest_1' } + ); + + // Redelivery through the OTHER instance: it reads the claim, which named + // a position the resume did not land at. + const second = await resume(runId, hook, 'resume_1', 'digest_1', payload); + + expect(second.event.eventId).toBe(first.event.eventId); + expect(await countHookReceived(runId)).toBe(1); + }); + + it('converges a redelivery whose claim still names the occupied position', async () => { + const { runId, hook } = await setup(); + const other = await seedStaleAllocator(runId); + const payload = new Uint8Array([1, 2, 3]); + + const first = await other.events.create( + runId, + { + eventType: 'hook_received', + specVersion: SPEC_VERSION_CURRENT, + correlationId: hook.hookId, + eventData: { token: hook.token, payload }, + }, + { resumeId: 'resume_1', resumePayloadDigest: 'digest_1' } + ); + + // Roll the claim back to the position it named before the append, as a + // crash between the append and the claim's correction would leave it. + // That position holds an unrelated event, so a reader that trusts the + // claim reports an `attr_set` as this resume's event: no error, no + // second event, and the payload silently gone. + const claimPath = hookResumeClaimPath(testDir, runId, 'resume_1'); + const claim = JSON.parse(await fs.readFile(claimPath, 'utf8')); + const occupant = (await storage.events.list({ runId })).data.find( + (event) => event.eventType === 'attr_set' + ); + await fs.writeFile( + claimPath, + JSON.stringify({ ...claim, eventId: occupant?.eventId }) + ); + + const second = await resume(runId, hook, 'resume_1', 'digest_1', payload); + + expect(second.event.eventType).toBe('hook_received'); + expect(second.event.eventId).toBe(first.event.eventId); + expect(await countHookReceived(runId)).toBe(1); + }); + }); + it('rejects a reused resumeId + digest that belongs to a DIFFERENT hook', async () => { const { runId, hook } = await setup(); const otherHook = await createHook(storage, runId, { diff --git a/packages/world-local/src/storage/slot-identity.test.ts b/packages/world-local/src/storage/slot-identity.test.ts new file mode 100644 index 0000000000..7ef644e724 --- /dev/null +++ b/packages/world-local/src/storage/slot-identity.test.ts @@ -0,0 +1,469 @@ +import fs from 'node:fs/promises'; +import os from 'node:os'; +import path from 'node:path'; +import { + EVENT_ID_BODY_LENGTH, + EVENT_ID_PREFIX, + eventIdToSlot, + FIRST_EVENT_SLOT, + SPEC_VERSION_CURRENT, +} from '@workflow/world'; +import { afterEach, beforeEach, describe, expect, it } from 'vitest'; +import { SORT_KEY_CURSOR_PREFIX } from '../fs.js'; +import { createStorage } from '../storage.js'; +import { monotonicUlid } from './helpers.js'; + +let testDir: string; +let storage: ReturnType; + +beforeEach(async () => { + testDir = await fs.mkdtemp(path.join(os.tmpdir(), 'wl-slot-')); + storage = createStorage(testDir); +}); + +afterEach(async () => { + await fs.rm(testDir, { recursive: true, force: true }); +}); + +const serialized = (value: unknown) => + ({ data: JSON.stringify(value), encoding: 'json' }) as any; + +function slotId(slot: number): string { + return `${EVENT_ID_PREFIX}${String(slot).padStart(EVENT_ID_BODY_LENGTH, '0')}`; +} + +async function startRun(): Promise { + const created = await storage.events.create('', { + eventType: 'run_created', + specVersion: SPEC_VERSION_CURRENT, + eventData: { + deploymentId: 'dpl_slot', + workflowName: 'slotWorkflow', + input: serialized([]), + }, + } as any); + const { runId } = created.event; + await storage.events.create(runId, { + eventType: 'run_started', + specVersion: SPEC_VERSION_CURRENT, + } as any); + return runId; +} + +async function listEventIds(runId: string): Promise { + const result = await storage.events.list({ + runId, + pagination: { limit: 1000 }, + }); + return result.data.map((event) => event.eventId); +} + +function slotsOf(eventIds: string[]): (number | null)[] { + return eventIds.map((eventId) => eventIdToSlot(eventId)); +} + +describe('slot event ids', () => { + it('numbers a new run densely from the first slot, in log order', async () => { + const runId = await startRun(); + for (let i = 0; i < 5; i++) { + await storage.events.create(runId, { + eventType: 'step_created', + correlationId: `step_${i}`, + specVersion: SPEC_VERSION_CURRENT, + eventData: { stepName: `step${i}`, input: serialized([]) }, + } as any); + } + + const eventIds = await listEventIds(runId); + // run_created, run_started, then one step_created each. `events.list` + // returns chronological order, so the slots must come out sorted and + // gapless starting at the first slot. + expect(eventIds).toEqual( + eventIds.map((_, i) => slotId(FIRST_EVENT_SLOT + i)) + ); + }); + + it('stays dense when writers race for the same slot', async () => { + const runId = await startRun(); + const width = 20; + await Promise.all( + Array.from({ length: width }, (_, i) => + storage.events.create(runId, { + eventType: 'step_created', + correlationId: `step_${i}`, + specVersion: SPEC_VERSION_CURRENT, + eventData: { stepName: `step${i}`, input: serialized([]) }, + } as any) + ) + ); + + // Every writer starts from the same view of the log, so all but one lose + // the publish and bump. Bump-and-report means none of them fail, and the + // log they produce is still gapless. + const slots = slotsOf(await listEventIds(runId)); + expect(slots).toEqual( + Array.from({ length: width + 2 }, (_, i) => FIRST_EVENT_SLOT + i) + ); + }); + + it('leaves no hole behind writes that are rejected', async () => { + const runId = await startRun(); + const width = 20; + // Every writer claims the same correlation id, so exactly one + // step_created survives the entity-creation dedup and the rest are + // rejected. A slot + // drawn before the publish and never handed back would be burned by each + // rejection, and allocation only moves forward, so every such hole is + // permanent. + const results = await Promise.allSettled( + Array.from({ length: width }, () => + storage.events.create(runId, { + eventType: 'step_created', + correlationId: 'step_contended', + specVersion: SPEC_VERSION_CURRENT, + eventData: { stepName: 'contended', input: serialized([]) }, + } as any) + ) + ); + expect(results.filter((r) => r.status === 'fulfilled')).toHaveLength(1); + + // The next write is what exposes a burned slot: it lands right behind the + // winner if every rejection gave its slot back, and `width - 1` positions + // past it if none of them did. + await storage.events.create(runId, { + eventType: 'step_created', + correlationId: 'step_after', + specVersion: SPEC_VERSION_CURRENT, + eventData: { stepName: 'after', input: serialized([]) }, + } as any); + + const slots = slotsOf(await listEventIds(runId)); + // run_created, run_started, the one step_created that won, and the write + // that followed it. + expect(slots).toEqual([ + FIRST_EVENT_SLOT, + FIRST_EVENT_SLOT + 1, + FIRST_EVENT_SLOT + 2, + FIRST_EVENT_SLOT + 3, + ]); + }); + + it('leaves no hole when a rejected write is overtaken by another', async () => { + const runId = await startRun(); + const width = 8; + for (let i = 0; i < width; i++) { + await storage.events.create(runId, { + eventType: 'step_started', + correlationId: `step_${i}`, + specVersion: SPEC_VERSION_CURRENT, + eventData: { stepName: `step${i}`, input: serialized([]) }, + } as any); + } + + // Each duplicate names a different step, so they take different per-step + // locks and their draws interleave. This is the shape a step storm + // produces: several replays of one run each re-issuing a step_started the + // winner already published. A slot reserved at the draw and handed back + // only when it is still the highest one drawn cannot survive this β€” by the + // time a rejection lands, the next writer has drawn past it. + const results = await Promise.allSettled( + Array.from({ length: width }, (_, i) => + storage.events.create(runId, { + eventType: 'step_started', + correlationId: `step_${i}`, + specVersion: SPEC_VERSION_CURRENT, + eventData: { stepName: `step${i}`, input: serialized([]) }, + } as any) + ) + ); + expect(results.filter((r) => r.status === 'fulfilled')).toHaveLength(0); + + await storage.events.create(runId, { + eventType: 'step_created', + correlationId: 'step_after', + specVersion: SPEC_VERSION_CURRENT, + eventData: { stepName: 'after', input: serialized([]) }, + } as any); + + // run_created, run_started, a step_created + step_started per step, and + // the write that followed the rejections. + const slots = slotsOf(await listEventIds(runId)); + expect(slots).toEqual( + Array.from({ length: width * 2 + 3 }, (_, i) => FIRST_EVENT_SLOT + i) + ); + }); + + it('orders a terminal event after every event it raced', async () => { + const runId = await startRun(); + await storage.events.create(runId, { + eventType: 'step_created', + correlationId: 'step_a', + specVersion: SPEC_VERSION_CURRENT, + eventData: { stepName: 'a', input: serialized([]) }, + } as any); + await storage.events.create(runId, { + eventType: 'run_completed', + specVersion: SPEC_VERSION_CURRENT, + eventData: { output: serialized('done') }, + } as any); + + const eventIds = await listEventIds(runId); + expect(eventIds.at(-1)).toBe(slotId(eventIds.length)); + }); + + it('numbers a lazily created step_created from the same allocator', async () => { + const runId = await startRun(); + // A step_started carrying the creation payload with no step_created ahead + // of it makes the World synthesize one. That synthetic event is the only + // event the World writes without a caller asking for it by name, so it is + // the one place a second id scheme can leak into a slot-numbered log. + await storage.events.create(runId, { + eventType: 'step_started', + correlationId: 'step_lazy', + specVersion: SPEC_VERSION_CURRENT, + eventData: { stepName: 'lazy', input: serialized([]) }, + } as any); + + const eventIds = await listEventIds(runId); + expect(slotsOf(eventIds)).toEqual( + Array.from({ length: eventIds.length }, (_, i) => FIRST_EVENT_SLOT + i) + ); + // A ULID id here has no sort key, so `events.list` would return it on + // every page and the cursor would eventually repeat. + const events = await storage.events.list({ + runId, + pagination: { limit: 1000 }, + }); + // The synthetic step_created takes the lower slot: the step_started that + // triggered it holds a candidate, not a reservation, so publishing the + // step_created first pushes the step_started up one. Replay reads them in + // the order they happened. + expect(events.data.map((event) => event.eventType)).toEqual([ + 'run_created', + 'run_started', + 'step_created', + 'step_started', + ]); + }); + + it('paginates a run whose step_created events were created lazily', async () => { + const runId = await startRun(); + for (let i = 0; i < 6; i++) { + await storage.events.create(runId, { + eventType: 'step_started', + correlationId: `step_lazy_${i}`, + specVersion: SPEC_VERSION_CURRENT, + eventData: { stepName: `lazy${i}`, input: serialized([]) }, + } as any); + } + + // Walk the log the way the runtime does: one page at a time, asserting the + // cursor always advances. A mixed-scheme log stalls here rather than at + // the id assertion above. + const seenCursors = new Set(); + const walked: string[] = []; + let cursor: string | undefined; + for (let page = 0; page < 20; page++) { + const result = await storage.events.list({ + runId, + pagination: { limit: 3, sortOrder: 'asc', cursor }, + }); + walked.push(...result.data.map((event) => event.eventId)); + if (!result.hasMore) break; + expect(result.cursor).toBeTruthy(); + expect(seenCursors.has(result.cursor as string)).toBe(false); + seenCursors.add(result.cursor as string); + cursor = result.cursor as string; + } + + expect(walked).toEqual(await listEventIds(runId)); + expect(slotsOf(walked)).toEqual( + Array.from({ length: walked.length }, (_, i) => FIRST_EVENT_SLOT + i) + ); + }); + + it('keeps a ULID-numbered run on ULIDs', async () => { + const runId = await startRun(); + // Rewrite the run's log the way it would look had it been created before + // slot ids existed. The scheme is pinned by what is on disk, not by a + // stored flag, so this is the whole of the upgrade path. + const eventsDir = path.join(testDir, 'events'); + const files = (await fs.readdir(eventsDir)).filter((file) => + file.startsWith(`${runId}-`) + ); + files.sort(); + for (const file of files) { + const legacyId = `${EVENT_ID_PREFIX}${monotonicUlid()}`; + const raw = await fs.readFile(path.join(eventsDir, file), 'utf8'); + await fs.writeFile( + path.join(eventsDir, `${runId}-${legacyId}.json`), + raw.replace(/"eventId": "evnt_[^"]+"/, `"eventId": "${legacyId}"`) + ); + await fs.rm(path.join(eventsDir, file)); + } + // The allocator memoizes each run's scheme, so drop the cache the way a + // fresh process would see it. + storage.events.clearCache?.(); + + await storage.events.create(runId, { + eventType: 'step_created', + correlationId: 'step_after_upgrade', + specVersion: SPEC_VERSION_CURRENT, + eventData: { stepName: 'afterUpgrade', input: serialized([]) }, + } as any); + + const eventIds = await listEventIds(runId); + expect(eventIds).toHaveLength(3); + // No slot ids anywhere: one slot id in a ULID log would sort before every + // ULID (its body starts with ten zeros) and replay out of order. + expect(slotsOf(eventIds)).toEqual([null, null, null]); + }); +}); + +describe('skipped-slot report', () => { + /** Writes `count` step_created events, returning the slots they landed on. */ + async function fill(runId: string, count: number): Promise { + const slots: number[] = []; + for (let i = 0; i < count; i++) { + const result = await storage.events.create(runId, { + eventType: 'step_created', + correlationId: `filler_${i}`, + specVersion: SPEC_VERSION_CURRENT, + eventData: { stepName: `filler${i}`, input: serialized([]) }, + } as any); + slots.push(eventIdToSlot(result.event.eventId) as number); + } + return slots; + } + + it('hands back the events occupying the slots the write skipped', async () => { + const runId = await startRun(); + const stale = FIRST_EVENT_SLOT + 1; // what the run had after run_started + const filled = await fill(runId, 3); + + // A writer whose loaded log stopped at run_started asks for the slot right + // above it and is bumped past everything written since. + const result = await storage.events.create( + runId, + { + eventType: 'wait_created', + correlationId: 'wait_a', + specVersion: SPEC_VERSION_CURRENT, + eventData: { resumeAt: new Date(0).toISOString() }, + } as any, + { eventCount: stale } + ); + + expect(eventIdToSlot(result.event.eventId)).toBe(stale + filled.length + 1); + expect(result.events?.map((event) => event.eventId)).toEqual( + filled.map(slotId) + ); + expect(result.hasMore).toBe(false); + }); + + it('reports nothing when the write lands on the slot it asked for', async () => { + const runId = await startRun(); + const result = await storage.events.create( + runId, + { + eventType: 'wait_created', + correlationId: 'wait_a', + specVersion: SPEC_VERSION_CURRENT, + eventData: { resumeAt: new Date(0).toISOString() }, + } as any, + { eventCount: FIRST_EVENT_SLOT + 1 } + ); + + expect(eventIdToSlot(result.event.eventId)).toBe(FIRST_EVENT_SLOT + 2); + expect(result.events).toBeUndefined(); + }); + + it('reports nothing when the writer sends no count', async () => { + const runId = await startRun(); + await fill(runId, 2); + const result = await storage.events.create(runId, { + eventType: 'wait_created', + correlationId: 'wait_a', + specVersion: SPEC_VERSION_CURRENT, + eventData: { resumeAt: new Date(0).toISOString() }, + } as any); + + expect(result.events).toBeUndefined(); + }); + + it('lets the sinceCursor delta answer when the writer asks for both', async () => { + // `sinceCursor` and `eventCount` both report through + // `events`/`cursor`/`hasMore`, and the runtime sends both on the same + // write. The delta is a strict superset of the skipped span (the skipped + // slots are all above the cursor) and, unlike the report, it advances + // `cursor`. Returning the narrower set alongside the delta's cursor would + // tell the caller it has read up to the delta end while handing it only + // part of that range, and the events in between would never be fetched + // again. + const runId = await startRun(); + const stale = FIRST_EVENT_SLOT + 1; + const filled = await fill(runId, 3); + + const result = await storage.events.create( + runId, + { + eventType: 'wait_created', + correlationId: 'wait_a', + specVersion: SPEC_VERSION_CURRENT, + eventData: { resumeAt: new Date(0).toISOString() }, + } as any, + { + eventCount: stale, + sinceCursor: `${SORT_KEY_CURSOR_PREFIX}${slotId(stale)}`, + } + ); + + const committed = result.event.eventId; + expect(eventIdToSlot(committed)).toBe(stale + filled.length + 1); + // Everything after the cursor, this write's own event included. + expect(result.events?.map((event) => event.eventId)).toEqual([ + ...filled.map(slotId), + committed, + ]); + expect(result.cursor).toBe(`${SORT_KEY_CURSOR_PREFIX}${committed}`); + expect(result.hasMore).toBe(false); + }); + + it('gives every racing writer the events it was decided without', async () => { + const runId = await startRun(); + const stale = FIRST_EVENT_SLOT + 1; + const width = 8; + + // All eight start from the same view, so seven of them are bumped and each + // one's report covers exactly the slots between `stale` and where it + // landed. Under contention the report can be a lower bound: a writer + // holding a lower slot may not have published yet, which `hasMore` says. + const results = await Promise.all( + Array.from({ length: width }, (_, i) => + storage.events.create( + runId, + { + eventType: 'step_created', + correlationId: `racer_${i}`, + specVersion: SPEC_VERSION_CURRENT, + eventData: { stepName: `racer${i}`, input: serialized([]) }, + } as any, + { eventCount: stale } + ) + ) + ); + + for (const result of results) { + const landed = eventIdToSlot(result.event.eventId) as number; + const reported = result.events ?? []; + const span = landed - stale - 1; + expect(reported.length).toBeLessThanOrEqual(span); + expect(result.hasMore ?? false).toBe(reported.length < span); + for (const event of reported) { + const slot = eventIdToSlot(event.eventId) as number; + expect(slot).toBeGreaterThan(stale); + expect(slot).toBeLessThan(landed); + } + } + }); +}); diff --git a/packages/world-postgres/src/drizzle/migrations/0019_add_event_slots.sql b/packages/world-postgres/src/drizzle/migrations/0019_add_event_slots.sql new file mode 100644 index 0000000000..99b7db3a33 --- /dev/null +++ b/packages/world-postgres/src/drizzle/migrations/0019_add_event_slots.sql @@ -0,0 +1,74 @@ +-- Event ids become per-run slot positions (`evnt_` + a zero-padded decimal), +-- so an id is only unique together with its run. Runs created before this keep +-- their globally-unique ULIDs, which the composite key also admits. +-- +-- LOCKING. Replacing a primary key takes ACCESS EXCLUSIVE on +-- `workflow.workflow_events`, which blocks reads as well as writes, and the +-- migrator runs every pending migration in one transaction, so the lock is held +-- until all of them commit. Building the new key's index under that lock is the +-- part that grows with the table. On an empty or modest table this is +-- instantaneous and needs no thought. +-- +-- On a large existing table, build the index first, outside the migrator, and +-- this migration adopts it instead of building its own: +-- +-- CREATE UNIQUE INDEX CONCURRENTLY "workflow_events_run_id_id_idx" +-- ON "workflow"."workflow_events" ("run_id", "id"); +-- +-- `CONCURRENTLY` cannot appear in this file: Postgres rejects it inside a +-- transaction block. Run it by hand, confirm the index came out valid, then +-- migrate. The branch below picks it up, and the exclusive lock then covers +-- only a catalog update rather than a full build. Adopting an index renames it +-- to the constraint's name, so it ends up as `workflow_events_run_id_id_pk` +-- either way and the two paths leave the same schema behind. +-- Bound the wait for that lock. A pending ACCESS EXCLUSIVE queues ahead of +-- every lock request that arrives after it, so waiting on one long-running +-- reader stalls all traffic to the table for as long as the wait lasts. Ten +-- seconds of that is a blip; an unbounded wait is an outage. Failing instead +-- leaves the migration unapplied and retryable. Raise it by hand for a +-- maintenance window. +-- +-- `SET LOCAL` lasts for the transaction, and the migrator runs every pending +-- migration in one, so a migration that follows this one in the same batch +-- inherits the timeout. +SET LOCAL lock_timeout = '10s';--> statement-breakpoint +ALTER TABLE "workflow"."workflow_events" DROP CONSTRAINT "workflow_events_pkey";--> statement-breakpoint +DO $$ +BEGIN + IF EXISTS ( + SELECT 1 + FROM pg_class c + JOIN pg_namespace n ON n.oid = c.relnamespace + JOIN pg_index i ON i.indexrelid = c.oid + WHERE n.nspname = 'workflow' + AND c.relname = 'workflow_events_run_id_id_idx' + AND c.relkind = 'i' + AND i.indisunique + AND i.indisvalid + ) THEN + ALTER TABLE "workflow"."workflow_events" + ADD CONSTRAINT "workflow_events_run_id_id_pk" + PRIMARY KEY USING INDEX "workflow_events_run_id_id_idx"; + ELSE + -- A `CREATE UNIQUE INDEX CONCURRENTLY` that failed leaves an index of this + -- name behind marked invalid. The branch above rejects it, and the key + -- built here is a different index under a different name, so without this + -- the invalid one would survive the migration: never used by a plan, still + -- maintained on every insert. Dropping it also makes a second attempt at + -- the concurrent build possible without a manual cleanup first. + DROP INDEX IF EXISTS "workflow"."workflow_events_run_id_id_idx"; + ALTER TABLE "workflow"."workflow_events" + ADD CONSTRAINT "workflow_events_run_id_id_pk" PRIMARY KEY ("run_id","id"); + END IF; +END $$;--> statement-breakpoint +-- Redundant once the primary key leads with `run_id`: that index serves every +-- by-run lookup and range scan this one did, so keeping it only costs a second +-- write per event on the table's hottest path. +DROP INDEX IF EXISTS "workflow"."workflow_events_run_id_index";--> statement-breakpoint +-- One row per slot-numbered run. Its absence is the "this run predates slots" +-- signal, so no backfill: existing runs stay on ULIDs for the rest of their +-- lives. A marker only: positions are allocated by the insert that occupies +-- them, read from the event log itself. +CREATE TABLE IF NOT EXISTS "workflow"."workflow_event_slots" ( + "run_id" varchar PRIMARY KEY NOT NULL +); diff --git a/packages/world-postgres/src/drizzle/migrations/meta/_journal.json b/packages/world-postgres/src/drizzle/migrations/meta/_journal.json index 4ce969faa2..7df53dba4e 100644 --- a/packages/world-postgres/src/drizzle/migrations/meta/_journal.json +++ b/packages/world-postgres/src/drizzle/migrations/meta/_journal.json @@ -134,6 +134,13 @@ "when": 1785619990000, "tag": "0018_add_hook_token_retention", "breakpoints": true + }, + { + "idx": 19, + "version": "7", + "when": 1786060800000, + "tag": "0019_add_event_slots", + "breakpoints": true } ] } diff --git a/packages/world-postgres/src/drizzle/schema.ts b/packages/world-postgres/src/drizzle/schema.ts index 578c78cf2d..c12f28e087 100644 --- a/packages/world-postgres/src/drizzle/schema.ts +++ b/packages/world-postgres/src/drizzle/schema.ts @@ -133,7 +133,7 @@ export const runs = schema.table( export const events = schema.table( 'workflow_events', { - eventId: varchar('id').primaryKey(), + eventId: varchar('id').notNull(), eventType: varchar('type').$type().notNull(), correlationId: varchar('correlation_id'), createdAt: timestamp('created_at').defaultNow().notNull(), @@ -153,7 +153,14 @@ export const events = schema.table( > >, (tb) => [ - index().on(tb.runId), + // Event ids are per-run slot positions, so `evnt_…0001` exists once per + // run and is only unique together with the run it belongs to. Runs + // created before slots keep globally-unique ULIDs, which this key also + // admits. + primaryKey({ columns: [tb.runId, tb.eventId] }), + // No standalone index on `runId`: the primary key leads with it, so every + // by-run lookup and range scan is served by that index already. Keeping one + // would cost a second write per event on the table's hottest path. index().on(tb.correlationId), // Runtime-correlated one-shot events must be unique per (run, correlation) // β€” without @@ -170,6 +177,21 @@ export const events = schema.table( ] ); +/** + * Which runs are slot-numbered. A row exists iff the run is, so its absence is + * exactly the "this run predates slots, keep minting ULIDs" signal β€” no scan of + * the event log is needed to tell the two schemes apart. + * + * A marker, not a counter. Positions are allocated by the insert that occupies + * them (`MAX(slot) + 1` read from the log inside the INSERT), so nothing is + * handed out ahead of the write that uses it and a write that fails leaves the + * position free for the next one. A counter here would instead burn a position + * per failed write, and every such hole is permanent. + */ +export const eventSlots = schema.table('workflow_event_slots', { + runId: varchar('run_id').primaryKey(), +}); + export const steps = schema.table( 'workflow_steps', { diff --git a/packages/world-postgres/src/index.ts b/packages/world-postgres/src/index.ts index 463ba42fba..dae35f39cc 100644 --- a/packages/world-postgres/src/index.ts +++ b/packages/world-postgres/src/index.ts @@ -64,7 +64,13 @@ export function createWorld( return { specVersion: SPEC_VERSION_CURRENT, - capabilities: { hookRetention: { active: true } }, + capabilities: { + hookRetention: { active: true }, + // New runs get dense per-run slot event ids. Runs created before this + // keep their ULIDs; the scheme is pinned by whether the run owns a slot + // counter, not by this flag, which only says what new runs get. + slotEventIds: true, + }, ...storage, ...streamer, ...queue, diff --git a/packages/world-postgres/src/storage.ts b/packages/world-postgres/src/storage.ts index cecbc51ba6..7d6c10f86d 100644 --- a/packages/world-postgres/src/storage.ts +++ b/packages/world-postgres/src/storage.ts @@ -31,7 +31,11 @@ import type { import { ATTRIBUTE_MAX_PER_RUN, AttributeValidationError, + EVENT_ID_BODY_LENGTH, + EVENT_ID_PREFIX, EventSchema, + eventIdToSlot, + FIRST_EVENT_SLOT, getMaxEventsPerRun, HookSchema, isChildEntityCreationEvent, @@ -44,6 +48,7 @@ import { requiresNewerWorld, SPEC_VERSION_CURRENT, StepSchema, + slotToEventId, stripEventDataRefs, TERMINAL_STEP_STATUSES, TERMINAL_WORKFLOW_RUN_STATUSES, @@ -65,6 +70,7 @@ import { notExists, notInArray, or, + type SQL, sql, } from 'drizzle-orm'; import { monotonicFactory } from 'ulid'; @@ -74,6 +80,245 @@ import { compact } from './util.js'; const DAY_MS = 24 * 60 * 60 * 1000; +/** + * A drizzle handle, either the pool or a transaction. Slot allocation runs on + * whichever one the caller is already inside, so the position an insert takes + * commits or rolls back with the insert itself. + */ +type DrizzleLike = Pick; + +/** Only for legacy (pre-slot) runs; see `allocateEventId`. */ +const legacyEventUlid = monotonicFactory(); + +/** + * How many positions one insert will try before giving up. Reached only when a + * run is taking concurrent writes faster than any of them can commit. + */ +const SLOT_INSERT_MAX_ATTEMPTS = 40; +/** + * Collisions that retry the instant the conflicting writer settles. + * + * `ON CONFLICT DO NOTHING` does not skip an uncommitted conflicting row: the + * unique-index check waits on that writer's transaction and only then reports + * the conflict, so a lost race has already waited for exactly the thing the + * next position depends on. Sleeping on top of that adds latency to a + * suspension flush and buys nothing. + * + * The backoff below covers the shape blocking does not: writers that keep + * arriving while the loop spins, where jittering the herd is the only way the + * loop converges before it exhausts its attempts. + */ +const SLOT_INSERT_IMMEDIATE_ATTEMPTS = 8; +/** Backoff between collisions, so a wide fan-out spreads rather than lockstep. */ +const SLOT_INSERT_BASE_DELAY_MS = 2; +const SLOT_INSERT_MAX_DELAY_MS = 40; + +/** + * Isolation for every transaction an event insert can run inside. + * + * {@link insertEventRow} answers a collision by recomputing the next position + * and inserting again, which only terminates if the retry can see rows + * committed since the transaction began. Under REPEATABLE READ or SERIALIZABLE + * it cannot: every attempt reads the transaction's original snapshot, computes + * the same taken position, and the loop runs to its limit and 503s. READ + * COMMITTED is Postgres' default, so this is a statement of the requirement + * rather than a change, and it keeps a database whose + * `default_transaction_isolation` was raised from turning event writes into + * timeouts. Inserts outside a transaction need nothing: a lone statement takes + * a fresh snapshot at every isolation level. + */ +const SLOT_INSERT_TRANSACTION = { isolationLevel: 'read committed' } as const; + +/** The pg error behind a drizzle wrapper, or an empty shape if there is none. */ +function pgErrorOf(err: unknown): { code?: string; constraint?: string } { + const direct = err as { code?: string; constraint?: string }; + if (direct?.code) { + return direct; + } + return ( + (err as { cause?: { code?: string; constraint?: string } })?.cause ?? {} + ); +} + +/** + * The position a slot-numbered insert takes: one above the highest the run + * already holds, read inside the INSERT that takes it. + * + * Nothing hands out a position ahead of the write that fills it. A writer that + * loses a dedup race, or whose transaction rolls back, leaves the numbering + * untouched, so a log missing a position is missing an *event* rather than + * merely a number. The runtime depends on exactly that: it refuses to replay a + * log with a hole, because a position nothing occupies cannot be told apart + * from an event that never happened. + * + * A counter column would be cheaper and is what this used to be. It cannot + * hold that property: a number handed out before the write lands is a number + * lost whenever the write does not, and the resulting holes are permanent. + * + * The subquery is an index-only read of the primary key's last row for the + * run, not a scan. Ordering is lexicographic, which is the same order as by + * position because every body is zero-padded to a fixed width. + * + * Every numeric parameter is cast explicitly. `substring(text from $n)` with an + * untyped parameter resolves to the *regular expression* overload rather than + * the positional one, which quietly returns NULL for every id and hands every + * writer the first slot. + */ +function nextSlotId(runId: string): SQL { + const bodyFrom = sql.raw(String(EVENT_ID_PREFIX.length + 1)); + const width = sql.raw(String(EVENT_ID_BODY_LENGTH)); + const noEvents = sql.raw(String(FIRST_EVENT_SLOT - 1)); + return sql`${EVENT_ID_PREFIX} || lpad((coalesce((select cast(substring(prev.id from ${bodyFrom}) as bigint) from ${Schema.events} prev where prev.run_id = ${runId} order by prev.id desc limit 1), ${noEvents}) + 1)::text, ${width}, '0')`; +} + +/** + * The id an insert for `runId` should allocate with: a slot expression for a + * slot-numbered run, a fresh ULID for one that predates slots. + * + * A row in `workflow_event_slots` is the marker for the first case. Its + * absence is exactly the "this run predates slots" signal, which is why the + * table is still read even though nothing advances it any more. + * + * A legacy run keeps minting under the original `wevt_` prefix rather than + * moving to `evnt_`: a mid-life prefix change would sort every new event + * before every old one, since `evnt_` < `wevt_`. + */ +async function allocateEventId( + db: DrizzleLike, + runId: string +): Promise> { + const [row] = await db + .select({ runId: Schema.eventSlots.runId }) + .from(Schema.eventSlots) + .where(eq(Schema.eventSlots.runId, runId)) + .limit(1); + return row ? nextSlotId(runId) : `wevt_${legacyEventUlid()}`; +} + +/** + * Inserts one event row, retrying while the position it computed is taken. + * + * The primary-key conflict is absorbed by `ON CONFLICT DO NOTHING` rather than + * raised, so a lost race costs a retry instead of the enclosing transaction β€” + * an error inside a transaction would poison it, and these inserts run in one. + * Every other unique violation still raises, which is what lets callers + * translate a dedup conflict on `workflow_events_entity_creation_unique`. + * + * Returns `undefined` only for an id that is a plain string (a legacy ULID, or + * the reserved first slot), where a conflict is the caller's answer rather + * than something to retry. + */ +async function insertEventRow( + db: DrizzleLike, + values: Omit & { + eventId: string | SQL; + } +): Promise<{ eventId: string; createdAt: Date } | undefined> { + const runId = values.runId; + const allocates = typeof values.eventId !== 'string'; + for (let attempt = 0; ; attempt++) { + const [row] = await db + .insert(Schema.events) + .values(values as typeof Schema.events.$inferInsert) + .onConflictDoNothing({ + target: [Schema.events.runId, Schema.events.eventId], + }) + .returning({ + eventId: Schema.events.eventId, + createdAt: Schema.events.createdAt, + }); + if (row) { + return row; + } + if (!allocates || attempt >= SLOT_INSERT_MAX_ATTEMPTS) { + if (!allocates) { + return undefined; + } + throw new WorkflowWorldError( + `Could not allocate an event slot for run "${runId}" after ${SLOT_INSERT_MAX_ATTEMPTS} attempts`, + { status: 503 } + ); + } + if (attempt >= SLOT_INSERT_IMMEDIATE_ATTEMPTS) { + const delay = Math.min( + SLOT_INSERT_MAX_DELAY_MS, + SLOT_INSERT_BASE_DELAY_MS * + 2 ** (attempt - SLOT_INSERT_IMMEDIATE_ATTEMPTS) + ); + await new Promise((resolve) => + setTimeout(resolve, Math.random() * delay) + ); + } + } +} + +/** + * Marks a run being created as slot-numbered and returns its first event id. + * + * The row records the scheme and nothing else; positions come from the log + * itself, see {@link nextSlotId}. + * + * `DO NOTHING` on conflict because the arbitration that matters is the event + * insert: two writers racing one run_created both take the first slot, and the + * composite events primary key rejects the loser. + */ +async function openEventSlots(db: DrizzleLike, runId: string): Promise { + await db.insert(Schema.eventSlots).values({ runId }).onConflictDoNothing(); + return slotToEventId(FIRST_EVENT_SLOT); +} + +/** + * The report half of bump-and-report: the events sitting on the slots between + * the one the writer asked for and the one its write actually landed on. + * + * Returns `undefined` when there is nothing to report β€” the write took the slot + * it asked for, the run is not slot-numbered, or the caller sent a count from a + * log that is already ahead of this write. + * + * The set can be short of the slot span it covers. A position is taken by the + * INSERT that computes it, and that INSERT commits on its own, so at the moment + * this reads the span a concurrent writer holding a lower position may not have + * committed yet. Its row appears shortly after and no position is left behind, + * because a write that fails never took one. `hasMore` says the report is a + * lower bound for now rather than a permanent one, and it is advisory either + * way: the caller's ordinary incremental read still runs. + */ +async function reportSkippedSlots( + db: Drizzle, + runId: string, + committedEventId: string, + askedFor: number, + resolveData: ResolveData +): Promise<{ events: Event[]; hasMore: boolean } | undefined> { + const committedSlot = eventIdToSlot(committedEventId); + if ( + committedSlot === null || + askedFor < FIRST_EVENT_SLOT || + committedSlot <= askedFor + 1 + ) { + return undefined; + } + const rows = await db + .select() + .from(Schema.events) + .where( + and( + eq(Schema.events.runId, runId), + gt(Schema.events.eventId, slotToEventId(askedFor)), + lt(Schema.events.eventId, committedEventId) + ) + ) + .orderBy(Schema.events.eventId); + const events = rows.map((row) => { + row.eventData ||= row.eventDataJson; + return stripEventDataRefs(EventSchema.parse(compact(row)), resolveData); + }); + return { + events, + hasMore: events.length < committedSlot - askedFor - 1, + }; +} + function getHookRetentionLimitMs(): number { const days = Number( process.env.WORKFLOW_POSTGRES_HOOK_RETENTION_LIMIT_DAYS ?? 30 @@ -402,7 +647,7 @@ async function handleLegacyEventPostgres( ); } return insertLegacyEvent(tx); - }) + }, SLOT_INSERT_TRANSACTION) : await insertLegacyEvent(drizzle); const event = EventSchema.parse({ @@ -527,8 +772,18 @@ export function createEventsStorage(drizzle: Drizzle): Storage['events'] { ); } + // The id this call's event took, known only once its insert has + // committed: on a slot-numbered run the position is chosen inside the + // INSERT, so there is nothing to read before it. let eventId: string | undefined; - const getEventId = () => (eventId ??= `wevt_${ulid()}`); + // Lazy, because on a legacy run this mints a ULID and on a slot run it + // reads which of the two schemes applies. Every caller below awaits it + // immediately before its insert. A caller that has already fixed the id + // β€” run_created, which always takes the first slot β€” gets that back. + const getEventId = async ( + db: DrizzleLike = drizzle + ): Promise> => + eventId ?? (await allocateEventId(db, effectiveRunId)); // For run_created events, use client-provided runId or generate one server-side let effectiveRunId: string; @@ -646,7 +901,12 @@ export function createEventsStorage(drizzle: Drizzle): Storage['events'] { .returning(); if (inserted) { - const runCreatedEventId = `wevt_${ulid()}`; + // This synthetic run_created is the run's first event, so it + // opens the slot counter the rest of the run allocates from. + const runCreatedEventId = await openEventSlots( + drizzle, + effectiveRunId + ); await drizzle.insert(events).values({ runId: effectiveRunId, eventId: runCreatedEventId, @@ -696,12 +956,14 @@ export function createEventsStorage(drizzle: Drizzle): Storage['events'] { ); } - // Route to legacy handler for pre-event-sourcing runs + // Route to legacy handler for pre-event-sourcing runs. A run this old + // is ULID-numbered by definition, so the id is minted here rather than + // read out of a slot marker the run cannot have. if (isLegacySpecVersion(currentRun.specVersion)) { return handleLegacyEventPostgres( drizzle, effectiveRunId, - getEventId(), + `wevt_${legacyEventUlid()}`, data, currentRun, params @@ -740,23 +1002,24 @@ export function createEventsStorage(drizzle: Drizzle): Storage['events'] { .limit(1); // Create the event (still record it) - const [value] = await drizzle - .insert(Schema.events) - .values({ - runId: effectiveRunId, - eventId: getEventId(), - correlationId: data.correlationId, - eventType: data.eventType, - eventData: 'eventData' in data ? data.eventData : undefined, - specVersion: effectiveSpecVersion, - }) - .returning({ createdAt: Schema.events.createdAt }); + const value = await insertEventRow(drizzle, { + runId: effectiveRunId, + eventId: await getEventId(), + correlationId: data.correlationId, + eventType: data.eventType, + eventData: 'eventData' in data ? data.eventData : undefined, + specVersion: effectiveSpecVersion, + }); + if (!value) { + throw new EntityConflictError( + `run_cancelled for run "${effectiveRunId}" could not be created` + ); + } const result = { ...data, ...value, runId: effectiveRunId, - eventId: getEventId(), }; const parsed = EventSchema.parse(result); const resolveData = params?.resolveData ?? 'all'; @@ -930,6 +1193,10 @@ export function createEventsStorage(drizzle: Drizzle): Storage['events'] { `Workflow run "${effectiveRunId}" already exists` ); } + // Open the run's slot counter. Doing it here, rather than lazily on + // first allocation, is what makes "no row" mean "created before slots + // existed" for the rest of the run's life. + eventId = await openEventSlots(drizzle, effectiveRunId); run = deserializeRunError(compact(runValue)); } @@ -1269,12 +1536,10 @@ export function createEventsStorage(drizzle: Drizzle): Storage['events'] { // step_started. Because this synthetic event is in the same // transaction as the lazy step row and step_started event, we // cannot leave behind only one side of that materialization. - const stepCreatedEventId = `wevt_${ulid()}`; - await tx - .insert(events) - .values({ + try { + await insertEventRow(tx, { runId: effectiveRunId, - eventId: stepCreatedEventId, + eventId: await allocateEventId(tx, effectiveRunId), correlationId: data.correlationId, eventType: 'step_created', eventData: { @@ -1282,8 +1547,18 @@ export function createEventsStorage(drizzle: Drizzle): Storage['events'] { input: lazyData.input, }, specVersion: effectiveSpecVersion, - }) - .onConflictDoNothing(); + }); + } catch (err) { + // A concurrent writer already published this run's + // step_created for the same step. The event exists either way, + // which is all this synthetic write was for. + if ( + pgErrorOf(err).constraint !== + 'workflow_events_entity_creation_unique' + ) { + throw err; + } + } stepCreatedLazily = true; } @@ -1350,31 +1625,27 @@ export function createEventsStorage(drizzle: Drizzle): Storage['events'] { } } - // Allocate the step_started ULID only after the guarded step UPDATE - // has acquired and passed the row lock. Without a sequence, this is - // the local ordering guarantee we can provide: a writer blocked on - // the step row will not carry an older event id into a later insert. - const stepStartedEventId = `wevt_${ulid()}`; - eventId = stepStartedEventId; - const [eventValue] = await tx - .insert(events) - .values({ - runId: effectiveRunId, - eventId: stepStartedEventId, - correlationId: data.correlationId, - eventType: data.eventType, - eventData: storedEventData, - specVersion: effectiveSpecVersion, - }) - .returning({ createdAt: events.createdAt }); + // Allocate the step_started position only after the guarded step + // UPDATE has acquired and passed the row lock, so a writer blocked + // on the step row cannot carry an earlier position into a later + // insert. + const eventValue = await insertEventRow(tx, { + runId: effectiveRunId, + eventId: await allocateEventId(tx, effectiveRunId), + correlationId: data.correlationId, + eventType: data.eventType, + eventData: storedEventData, + specVersion: effectiveSpecVersion, + }); if (!eventValue) { throw new EntityConflictError( - `Event ${stepStartedEventId} could not be created` + `Event for step "${data.correlationId}" could not be created` ); } - return eventValue; - }); + eventId = eventValue.eventId; + return { createdAt: eventValue.createdAt }; + }, SLOT_INSERT_TRANSACTION); } // Handle step_completed event: update step status @@ -1572,25 +1843,22 @@ export function createEventsStorage(drizzle: Drizzle): Storage['events'] { token: eventData.token, conflictingRunId: existingHook.runId, }; - const conflictEventId = getEventId(); - - const [conflictValue] = await drizzle - .insert(events) - .values({ - runId: effectiveRunId, - eventId: conflictEventId, - correlationId: data.correlationId, - eventType: 'hook_conflict', - eventData: conflictEventData, - specVersion: effectiveSpecVersion, - }) - .returning({ createdAt: events.createdAt }); + const conflictValue = await insertEventRow(drizzle, { + runId: effectiveRunId, + eventId: await getEventId(), + correlationId: data.correlationId, + eventType: 'hook_conflict', + eventData: conflictEventData, + specVersion: effectiveSpecVersion, + }); if (!conflictValue) { throw new EntityConflictError( - `Event ${conflictEventId} could not be created` + `hook_conflict for run "${effectiveRunId}" could not be created` ); } + const conflictEventId = conflictValue.eventId; + eventId = conflictEventId; const conflictResult = { eventType: 'hook_conflict' as const, @@ -1690,31 +1958,27 @@ export function createEventsStorage(drizzle: Drizzle): Storage['events'] { ); } - // Allocate the ULID only after the row lock is acquired, + // Allocate the position only after the row lock is acquired, // matching step_started's ordering guarantee: a writer blocked - // on the run row must not carry an older event id into a later + // on the run row must not carry an earlier position into a later // insert. - const hookReceivedEventId = `wevt_${ulid()}`; - eventId = hookReceivedEventId; - const [eventValue] = await tx - .insert(events) - .values({ - runId: effectiveRunId, - eventId: hookReceivedEventId, - correlationId: data.correlationId, - eventType: data.eventType, - eventData: storedEventData, - specVersion: effectiveSpecVersion, - }) - .returning({ createdAt: events.createdAt }); + const eventValue = await insertEventRow(tx, { + runId: effectiveRunId, + eventId: await allocateEventId(tx, effectiveRunId), + correlationId: data.correlationId, + eventType: data.eventType, + eventData: storedEventData, + specVersion: effectiveSpecVersion, + }); if (!eventValue) { throw new EntityConflictError( - `Event ${hookReceivedEventId} could not be created` + `Event for hook "${data.correlationId}" could not be created` ); } - return eventValue; - }); + eventId = eventValue.eventId; + return { createdAt: eventValue.createdAt }; + }, SLOT_INSERT_TRANSACTION); } // Handle wait_created event: create wait entity @@ -1800,17 +2064,18 @@ export function createEventsStorage(drizzle: Drizzle): Storage['events'] { try { if (!value) { - [value] = await drizzle - .insert(events) - .values({ - runId: effectiveRunId, - eventId: getEventId(), - correlationId: data.correlationId, - eventType: data.eventType, - eventData: storedEventData, - specVersion: effectiveSpecVersion, - }) - .returning({ createdAt: events.createdAt }); + const inserted = await insertEventRow(drizzle, { + runId: effectiveRunId, + eventId: await getEventId(), + correlationId: data.correlationId, + eventType: data.eventType, + eventData: storedEventData, + specVersion: effectiveSpecVersion, + }); + if (inserted) { + eventId = inserted.eventId; + value = { createdAt: inserted.createdAt }; + } } } catch (err) { // Translate unique-violation on the correlated-event partial index @@ -1829,10 +2094,7 @@ export function createEventsStorage(drizzle: Drizzle): Storage['events'] { isChildEntityCreationEventType(data.eventType) || (data.eventType === 'attr_set' && data.eventData.writer.type === 'workflow'); - const pgErr = (err as { code?: string; constraint?: string }).code - ? (err as { code?: string; constraint?: string }) - : ((err as { cause?: { code?: string; constraint?: string } }) - .cause ?? {}); + const pgErr = pgErrorOf(err); const pgCode = pgErr.code; const pgConstraint = pgErr.constraint; if ( @@ -1846,16 +2108,16 @@ export function createEventsStorage(drizzle: Drizzle): Storage['events'] { } throw err; } - if (!value) { + if (!value || !eventId) { throw new EntityConflictError( - `Event ${getEventId()} could not be created` + `${data.eventType} for run "${effectiveRunId}" could not be created` ); } const result = { ...data, ...value, runId: effectiveRunId, - eventId: getEventId(), + eventId, ...(storedEventData !== undefined ? { eventData: storedEventData } : {}), @@ -1872,6 +2134,34 @@ export function createEventsStorage(drizzle: Drizzle): Storage['events'] { // For run_started: include all events so the runtime can skip // the initial events.list call and reduce TTFB. let eventPage: PaginatedResponse | undefined; + // The skipped-slot report and the inline delta below share + // `events`/`cursor`/`hasMore`, and the runtime sends both on the same + // write. The delta wins: the skipped slots all sit above the cursor, so + // it is a strict superset, and it is the only one of the two that + // advances `cursor`. Running the report anyway would cost a query whose + // result the delta overwrites. + if ( + params?.eventCount !== undefined && + typeof params.sinceCursor !== 'string' + ) { + const report = await reportSkippedSlots( + drizzle, + effectiveRunId, + parsed.eventId, + params.eventCount, + resolveData + ); + if (report) { + // Deliberately no cursor: the report is a lower bound on what this + // write skipped over, not a page the caller has now read to the end + // of, so it must not advance the caller's read position. + eventPage = { + data: report.events, + cursor: null, + hasMore: report.hasMore, + }; + } + } if (data.eventType === 'run_started' && run && !params?.skipPreload) { const eventRows = await drizzle .select() diff --git a/packages/world-postgres/test/storage.test.ts b/packages/world-postgres/test/storage.test.ts index 62f452b7c9..4882e17c00 100644 --- a/packages/world-postgres/test/storage.test.ts +++ b/packages/world-postgres/test/storage.test.ts @@ -6,11 +6,11 @@ import type { Step, WorkflowRun, } from '@workflow/world'; -import { SPEC_VERSION_CURRENT } from '@workflow/world'; +import { eventIdToSlot, SPEC_VERSION_CURRENT } from '@workflow/world'; import { encode } from 'cbor-x'; import { eq } from 'drizzle-orm'; import { Pool } from 'pg'; -import { decodeTime, ulid } from 'ulid'; +import { ulid } from 'ulid'; import { afterAll, afterEach, @@ -142,7 +142,7 @@ describe('Storage (Postgres integration)', () => { async function truncateTables() { await pool.query( - 'TRUNCATE TABLE workflow.workflow_events, workflow.workflow_steps, workflow.workflow_hooks, workflow.workflow_runs RESTART IDENTITY CASCADE' + 'TRUNCATE TABLE workflow.workflow_events, workflow.workflow_event_slots, workflow.workflow_steps, workflow.workflow_hooks, workflow.workflow_runs RESTART IDENTITY CASCADE' ); } @@ -814,7 +814,7 @@ describe('Storage (Postgres integration)', () => { expect(updated.attempt).toBe(1); // Incremented by step_started }); - it('allocates the step_started event id after the guarded step update', async () => { + it('allocates the step_started slot after the guarded step update', async () => { const stepId = 'step-start-lock'; await createStep(events, testRunId, { stepId, @@ -827,6 +827,14 @@ describe('Storage (Postgres integration)', () => { max: 1, }); const client = await lockPool.connect(); + // The suite's own pool is `max: 1`, so the parked step_started holds + // it for the duration. The overtaking writer needs a connection of + // its own, which is also the shape being tested: two processes. + const otherPool = new Pool({ + connectionString: container.getConnectionUri(), + max: 1, + }); + const otherEvents = createEventsStorage(createClient(otherPool)); try { await client.query('BEGIN'); @@ -841,19 +849,31 @@ describe('Storage (Postgres integration)', () => { }); await new Promise((resolve) => setTimeout(resolve, 50)); - const releasedAt = Date.now(); + // Written while step_started is still parked on the step row lock. + // A writer that drew its slot on entry would already hold a lower + // one than this; drawing after the lock puts it above. + const overtaking = await otherEvents.create(testRunId, { + eventType: 'step_created', + correlationId: 'step-start-lock-overtaker', + eventData: { stepName: 'test-step', input: new Uint8Array([1]) }, + }); await client.query('COMMIT'); const result = await started; - if (!result.event) { - throw new Error('Expected step_started event'); + if (!result.event || !overtaking.event) { + throw new Error('Expected both events'); } - expect( - decodeTime(result.event.eventId.slice('wevt_'.length)) - ).toBeGreaterThanOrEqual(releasedAt); + const startedSlot = eventIdToSlot(result.event.eventId); + const overtakingSlot = eventIdToSlot(overtaking.event.eventId); + expect(startedSlot).not.toBeNull(); + expect(overtakingSlot).not.toBeNull(); + expect(startedSlot as number).toBeGreaterThan( + overtakingSlot as number + ); } finally { client.release(); await lockPool.end(); + await otherPool.end(); } }); @@ -1121,7 +1141,7 @@ describe('Storage (Postgres integration)', () => { const result = await events.create(testRunId, eventData); expect(result.event.runId).toBe(testRunId); - expect(result.event.eventId).toMatch(/^wevt_/); + expect(result.event.eventId).toMatch(/^evnt_0{10}/); expect(result.event.eventType).toBe('step_started'); expect(result.event.correlationId).toBe('corr_123'); expect(result.event.createdAt).toBeInstanceOf(Date); @@ -1146,7 +1166,7 @@ describe('Storage (Postgres integration)', () => { }); expect(result.event.runId).toBe(testRunId); - expect(result.event.eventId).toMatch(/^wevt_/); + expect(result.event.eventId).toMatch(/^evnt_0{10}/); expect(result.event.eventType).toBe('step_failed'); expect(result.event.correlationId).toBe('corr_123_null'); expect(result.event.createdAt).toBeInstanceOf(Date); @@ -1812,6 +1832,243 @@ describe('Storage (Postgres integration)', () => { }); }); + describe('slot event ids', () => { + let testRunId: string; + beforeEach(async () => { + const run = await createRun(events, { + deploymentId: 'deployment-123', + workflowName: 'test-workflow', + input: new Uint8Array(), + }); + testRunId = run.runId; + }); + + it('numbers a run densely from the first slot', async () => { + await updateRun(events, testRunId, 'run_started'); + + const result = await events.list({ + runId: testRunId, + pagination: { sortOrder: 'asc' }, + }); + + expect(result.data.map((e) => eventIdToSlot(e.eventId))).toEqual([1, 2]); + }); + + it('gives concurrent writers distinct, dense slots', async () => { + const writers = 8; + // The suite's own pool is `max: 1`, which would serialize these writes + // and defeat the point. Give each writer a connection so they actually + // contend for the same slot. + const racePool = new Pool({ + connectionString: container.getConnectionUri(), + max: writers, + }); + const raceEvents = createEventsStorage(createClient(racePool)); + + try { + await Promise.all( + Array.from({ length: writers }, (_, i) => + raceEvents.create(testRunId, { + eventType: 'step_created', + correlationId: `slot-step-${i}`, + eventData: { + stepName: 'test-step', + input: new Uint8Array([i]), + }, + }) + ) + ); + } finally { + await racePool.end(); + } + + const result = await events.list({ + runId: testRunId, + pagination: { sortOrder: 'asc' }, + }); + const slots = result.data + .map((e) => eventIdToSlot(e.eventId)) + .sort((a, b) => (a ?? 0) - (b ?? 0)); + + // run_created holds slot 1 and the racing writers take the rest: no + // duplicate (the composite primary key rejects the loser, which retries) + // and no hole (nothing reserves a slot it does not then use), whatever + // order they happen to land in. + expect(slots).toEqual( + Array.from({ length: writers + 1 }, (_, i) => i + 1) + ); + }); + + it('leaves no hole behind writes that are rejected', async () => { + const writers = 8; + const racePool = new Pool({ + connectionString: container.getConnectionUri(), + max: writers, + }); + const raceEvents = createEventsStorage(createClient(racePool)); + + // Every writer claims the same correlation id, so exactly one + // step_created survives the entity-creation unique index and the rest + // are rejected with EntityConflictError. A slot handed out before the + // insert lands would be burned by each of those rejections, and a burned + // slot is a permanent hole: allocation only moves forward. + try { + const results = await Promise.allSettled( + Array.from({ length: writers }, () => + raceEvents.create(testRunId, { + eventType: 'step_created', + correlationId: 'slot-contended-step', + eventData: { + stepName: 'test-step', + input: new Uint8Array([1]), + }, + }) + ) + ); + expect(results.filter((r) => r.status === 'fulfilled')).toHaveLength(1); + } finally { + await racePool.end(); + } + + // The next write is what exposes a burned slot: it lands right behind + // the winner if no rejection consumed a position, and `writers - 1` + // past it if every rejection did. + await events.create(testRunId, { + eventType: 'step_created', + correlationId: 'slot-after-contention', + eventData: { stepName: 'test-step', input: new Uint8Array([1]) }, + }); + + const result = await events.list({ + runId: testRunId, + pagination: { sortOrder: 'asc' }, + }); + const slots = result.data + .map((e) => eventIdToSlot(e.eventId)) + .sort((a, b) => (a ?? 0) - (b ?? 0)); + + // run_created, the one step_created that won, and the write after it. + expect(slots).toEqual([1, 2, 3]); + }); + + it('hands back the events occupying the slots a write skipped', async () => { + await updateRun(events, testRunId, 'run_started'); + // What a writer that loaded the log right after run_started would report. + const stale = 2; + + for (let i = 0; i < 3; i++) { + await events.create(testRunId, { + eventType: 'step_created', + correlationId: `skipped-step-${i}`, + eventData: { stepName: 'test-step', input: new Uint8Array([i]) }, + }); + } + + const result = await events.create( + testRunId, + { + eventType: 'wait_created', + correlationId: 'skipped-wait', + eventData: { resumeAt: new Date('2099-01-01') }, + }, + { eventCount: stale } + ); + + expect(eventIdToSlot(result.event.eventId)).toBe(6); + expect(result.events?.map((e) => eventIdToSlot(e.eventId))).toEqual([ + 3, 4, 5, + ]); + expect(result.events?.map((e) => e.correlationId)).toEqual([ + 'skipped-step-0', + 'skipped-step-1', + 'skipped-step-2', + ]); + expect(result.hasMore).toBe(false); + }); + + it('reports nothing when the write lands on the slot it asked for', async () => { + const result = await events.create( + testRunId, + { + eventType: 'step_created', + correlationId: 'unskipped-step', + eventData: { stepName: 'test-step', input: new Uint8Array() }, + }, + { eventCount: 1 } + ); + + expect(eventIdToSlot(result.event.eventId)).toBe(2); + expect(result.events).toBeUndefined(); + }); + + it('reports nothing when the writer sends no count', async () => { + await updateRun(events, testRunId, 'run_started'); + await events.create(testRunId, { + eventType: 'step_created', + correlationId: 'uncounted-step', + eventData: { stepName: 'test-step', input: new Uint8Array() }, + }); + + const result = await events.create(testRunId, { + eventType: 'wait_created', + correlationId: 'uncounted-wait', + eventData: { resumeAt: new Date('2099-01-01') }, + }); + + expect(result.events).toBeUndefined(); + }); + + it('gives every racing writer the events it was decided without', async () => { + await updateRun(events, testRunId, 'run_started'); + const stale = 2; + const writers = 8; + const racePool = new Pool({ + connectionString: container.getConnectionUri(), + max: writers, + }); + const raceEvents = createEventsStorage(createClient(racePool)); + + let results: Awaited>[]; + try { + results = await Promise.all( + Array.from({ length: writers }, (_, i) => + raceEvents.create( + testRunId, + { + eventType: 'step_created', + correlationId: `race-report-${i}`, + eventData: { + stepName: 'test-step', + input: new Uint8Array([i]), + }, + }, + { eventCount: stale } + ) + ) + ); + } finally { + await racePool.end(); + } + + // Each writer's report covers only the slots between the one it asked + // for and the one it landed on. It can be short of that span: a writer + // holding a lower slot may not have committed its insert yet, which is + // what `hasMore` says. + for (const result of results) { + const landed = eventIdToSlot(result.event.eventId) as number; + const reported = result.events ?? []; + const span = landed - stale - 1; + expect(reported.length).toBeLessThanOrEqual(span); + expect(result.hasMore ?? false).toBe(reported.length < span); + for (const event of reported) { + const slot = eventIdToSlot(event.eventId) as number; + expect(slot).toBeGreaterThan(stale); + expect(slot).toBeLessThan(landed); + } + } + }); + }); + describe('concurrent entity-creation races', () => { let testRunId: string; beforeEach(async () => { diff --git a/packages/world-sim/DESIGN.md b/packages/world-sim/DESIGN.md new file mode 100644 index 0000000000..a973bb0b1f --- /dev/null +++ b/packages/world-sim/DESIGN.md @@ -0,0 +1,831 @@ +# world-sim: design + +How `@workflow/world-sim` is built and why it is built that way. The package +README is the introduction; this is the implementation. + +Two workspaces: + +| path | what it is | +|---|---| +| `packages/world-sim` | the World implementation, the scenario runner, the checkers | +| `workbench/sim-world` | the scenario book and the workflows it runs (`pnpm sim`) | + +--- + +## 1. Module map + +| module | responsibility | +|---|---| +| `world.ts` | the `World` implementation; wraps every method as a call point and attributes it to a writer | +| `store.ts` | in-memory event store β€” the event β†’ entity state machine, plus the write-time guards | +| `queue.ts` | deterministic queue: records messages, never delivers on its own | +| `clock.ts` | virtual clock; patches `Date.now()` readings, not timers | +| `ids.ts` | deterministic ULID minting from (virtual time, counter) | +| `drive.ts` | the scheduler loop and the scenario budgets | +| `tempo.ts` | the scripting layer: park / permit, and the wait bookkeeping under `runTo` | +| `writers.ts` | named writers and their level-triggered `runTo*` vocabulary | +| `scenario.ts` | runs one `ScenarioSpec` end to end and produces a `ScenarioResult` | +| `replay.ts` | cold-start replay verification of a finished log | +| `invariants.ts` | consistency checks re-derived from the event log alone | +| `report.ts` | renders a scenario; log positions for every event reference, colour only when the destination is a terminal | +| `streams.ts` | in-memory streamer | +| `build.ts` | bundles a project's workflows so the runtime can be handed real compiled code. Its own entry (`@workflow/world-sim/build`), because it reaches a compiler and playing a scenario should not | +| `load.ts` | loads a built bundle's flow handler β€” the half of the old `build.ts` that needs no compiler | +| `types.ts` | the public vocabulary | + +--- + +## 2. What the simulator has to model + +The design follows from four properties of the runtime. None of them are +choices this package made; they are the constraints it works inside. + +### The orchestrator is re-run from the top, every time + +A workflow function is not a coroutine that parks and resumes. It is +re-executed from its first line on every replay pass, inside a fresh `node:vm` +context (`createContext` in `packages/core/src/vm/index.ts`). The VM seals the +two obvious sources of nondeterminism: `Math.random` is seeded from +`${runId}:${workflowName}:${deploymentId}`, and `Date.now()` / `new Date()` +return a fixed timestamp advanced only from each consumed event's `createdAt`. + +A pass is therefore a pure function of (workflow code, run identity, event log +prefix). Stated precisely: **same log prefix β†’ same decisions.** That is the +whole basis of durability, and it is also the property the simulator exists to +attack β€” the interesting bug class is not "the workflow behaved randomly" but +"the decisions and the persisted log disagree", which requires the decisions to +have been made against a *different* log than the one that ended up durable. + +### Entity identity is positional + +Correlation ids come from `ctx.generateUlid()`, driven by the VM's seeded +`Math.random`. They are positional ordinals of one seeded sequence: the Nth +entity the workflow asks for gets the same id in every pass. Steps, hooks and +waits all draw from that one sequence. + +This is why a flipped branch is dangerous. It does not produce a different +step; it produces *a different step wearing the same name badge*. The runtime's +divergence check is a step-name comparison at the same ordinal: + +``` +Replay divergence: step event step_created for step_…445J belongs to +"…//settle", but the current step consumer is "…//recoverFirst" +``` + +### Suspension is the unit of progress + +`useStep` does the same thing on every pass: mint the correlation id, register +a `StepInvocationQueueItem`, subscribe a consumer, return a promise. What +differs is what the consumer finds. On replay the log holds +`step_created` / `step_started` / `step_completed` for that correlation id, so +the consumer hydrates the recorded result and the step body is never called. +First time through, the consumer reaches the end of the log, returns +`NotConsumed`, and the promise never resolves β€” so the workflow cannot proceed. + +When nothing can make further progress a `WorkflowSuspension` is raised +carrying the whole `invocationsQueue`. The runtime commits the pending +`*_created` events, executes what it can, and runs the workflow again from the +top against a longer log: + +``` +load log β†’ run workflow from top β†’ suspend β†’ commit + execute β†’ run from top β†’ … +``` + +Hooks follow the same shape with a different event family. `hook_created` is +committed at the next suspension rather than at the call. An out-of-band +`resumeHook(token, payload)` writes `hook_received` **and enqueues a flow +message**, so the run wakes up. Payloads landing before the workflow awaits are +buffered in a `payloadsQueue`, which is why a duplicate delivery is absorbed +rather than lost. + +There is no hook state a workflow can read β€” the surface is `token`, +`getConflict()`, `dispose()`, `then`, `[Symbol.asyncIterator]`. The only way to +observe a hook is to attach a continuation and see whether it resolves, which +is a *timing* observation, not a state read. That is why the scenario API +steers time rather than poking state. + +### Nothing sleeps + +`sleep()` registers a `WaitInvocationQueueItem`; `wait_created` records a +`resumeAt`; the runtime enqueues a delayed queue message. When that message is +delivered, the flow handler's "complete elapsed waits" pass compares +`Date.now() >= resumeAt` and writes `wait_completed`. + +A timeout is a delayed message plus a clock comparison. That is exactly why +virtual time works here, and why a thirty-day sleep costs microseconds. + +### Where the concurrency actually is + +The workflow body is single-threaded JS and stays that way. The interleaving +that matters lives in three other places: + +1. **Between passes.** The log grows. A branch decided in pass N was decided + against pass N's prefix. +2. **Between event deliveries inside one pass.** Several awaits can be pending + at once. The runtime forces resolution order to match log position via the + delivery-barrier registry (`registerDeliveryBarrier` / + `awaitEarlierDeliveries` in `private.ts`), so this one is reproducible. +3. **Between invocations.** Two flow deliveries for one run can execute + simultaneously in different processes. This is the one the SDK cannot make + deterministic by itself. + +Two step bodies that suspend together are already concurrent writers to one +log, inside a single delivery. That is cheaper to reach than "concurrent +writers" suggests, and it is the case the writer vocabulary is built around. + +--- + +## 3. Interception + +### Every World method is a call point + +Each method on the World is wrapped so a scenario can stop it. The wrapper +records the call, fires any matching watches, runs the underlying +implementation, fires the watches again on the way out, and only then resumes +the caller. + +A watch action returns a promise, and the intercepted call awaits it. That is +the entire hold mechanism: a "held" writer is a caller blocked inside a World +method. `release()` resolves that promise. + +### Two phases, and a third hold that is not one + +```ts +type CallPhase = 'before' | 'after'; +``` + +`before` and `after` bracket the call: + +| held at | what a competing write does | +|---|---| +| `before` | no position taken yet, so a write landing during the hold sorts **ahead** | +| `after` | durable, and the writer has not been resumed yet | + +`after` is the window the package was originally built for: "the hook arrives +after `step_started` is durable and before the workflow resumes". + +Neither phase produces the opposite order, and a write is not atomic, so the +opposite order is reachable: a real backend mints the event id *first* β€” +DynamoDB does not generate ids, and that id is the log's sort key β€” and only +then attempts the storage write. Between the two the event has a position but +no visibility, and a write that commits in that window sorts **behind** it. + +That gap is the point, not a detail. It is the only way to produce an event +*behind* a position a reader has already read past β€” a complete, consistent log +prefix that is simply missing an event still in flight. No high-water-mark fence +can represent that shape. + +It is not a phase, though, because the writer holding it is not blocked inside a +World method: the script owns the two halves explicitly, via `reservePosition` / +`withReservedPosition` under `sim.beginHookDelivery` (Β§Withholdings below). A +phase would have been a second way to say the same thing, and it went unused. + +### Watches do not fire inside watches + +Calls made from inside another watch's action are not call points. Without +that rule a watch on `events.create` would re-trigger on the `hook_received` +it just wrote, and every scenario using `deliverHook` would recurse forever. +The depth is tracked and surfaced in the trace, so a line committed from inside +a held call is visibly at depth > 0. + +A related rule is easy to get wrong: the depth counter must be raised only by +`asExternal`, which brackets exactly one call. Raising it for the whole +duration of a watch *action* is correct for something that returns immediately +and wrong for a hold, which does not return until the scenario releases it β€” +under that rule, holding one writer makes every other writer's call stop being +a call point, so a held step body's sibling becomes invisible and unsteerable. + +### Writer attribution is derived, not instrumented + +Writer identity comes from the intercepted call plus its request. No runtime +hook is needed: + +| write | writer | +|---|---| +| `step_created` / `hook_created` / `wait_created` / `run_*` / `attr_*` | orchestrator | +| `step_started` | orchestrator (the executor, which precedes the body) | +| `step_completed` / `step_failed` @ correlationId C | `step:` | +| `hook_received` | external | +| `wait_completed` | the wait-continuation delivery | +| `events.list` / `runs.get` | orchestrator (the read half) | + +The writer is printed as a column in every event stream, so a rendered log says +*who* wrote each line. + +--- + +## 4. Determinism machinery + +### Clock + +`install()` patches `Date.now()` and the zero-argument `Date` constructor to +read the virtual clock. Timers are deliberately **not** patched: +`@workflow/core` uses `setTimeout(fn, 0)` as a macrotask barrier in several +ordering-sensitive places (`events-consumer.ts`, `private.ts`), and swapping +those for fake timers would change the very interleavings the simulation exists +to observe. Real zero-delay timers stay real; only the *readings* of wall time +move. + +The clock never moves on its own. Only the scheduler calls `advanceTo` / +`advanceBy`, so two runs of a scenario see the same sequence of timestamps. + +### Ids + +Every id is a function of (virtual time, per-scenario counter) β€” never +`Math.random()` or the host clock. They still have to be real ULIDs, because +`@workflow/world` validates run ids with `z.string().ulid()` and decodes the +embedded timestamp, so the encoding is standard Crockford base32 with the +16 "random" characters filled from the counter. + +Byte-identical ids run to run are what make an event-stream dump usable as a +golden file. + +### Queue + +`@workflow/world-local`'s queue fires a detached delivery loop from inside +`queue()`, so a message races whatever the caller does next. Faithful to +production, useless for a simulation. + +Here `queue()` only *records*. Delivery happens when the scheduler asks, and it +always takes the same message: the minimum by `(readyAtMs, enqueueSeq)`. Delays +are virtual β€” a message 23 hours out is delivered by jumping the clock. +`ScenarioSpec.selectNext` can override the choice to pin an order the default +would not produce. + +### Scheduler + +``` +take the next message β†’ jump the clock to its delivery time β†’ hand it to the +flow handler β†’ wait β†’ repeat until the queue is empty or a budget stops it +``` + +Between deliveries the loop drains the event loop for several rounds +(`settle()`), because the runtime uses zero-delay macrotasks as ordering +barriers and `waitUntil`-style background work is not awaited by anyone. Without +that drain, a message enqueued from a trailing microtask would be missed and the +scenario would report a spurious stall. + +The scheduler lives apart from `scenario.ts` because two things drive it: a +scenario, and the replay verification that cold-starts a second world. + +**One delivery at a time.** This is the deliberate limit of the model β€” see +Β§9. + +--- + +## 5. The store + +A reference implementation of the World storage contract: the same event β†’ +entity state machine `@workflow/world-local` implements on the filesystem, +minus every mechanism that exists purely to make that state machine safe +against concurrent processes (exclusive-create claim files, per-entity locks, +staged/promoted hook events, canonical event-id pinning after a crash). One +delivery at a time in one process means those races cannot occur, and their +absence keeps the file small enough to audit. + +What is deliberately kept is every validation that *rejects* an event β€” +terminal-run guards, step lifecycle ordering, hook token uniqueness, wait +duplication. Those rejections are the observable contract the runtime is +written against; a simulation that relaxed them would agree with the runtime +about nothing interesting. + +### The two guards + +The fence is off by default and set per scenario +(`ScenarioSpec.preconditionGuard`, which flows into `SimWorldOptions`), which is +how a scenario can be run one flag apart from its neighbour. `countGuard` +**follows the fence** unless a spec says otherwise, because that is what +production does β€” see below. + +**`preconditionGuard`** models `WorldCapabilities.preconditionGuard`: reject a +replay-context write whose `stateUpdatedAt` snapshot predates the newest +externally-originated event. In the SDK this is declared by **world-vercel +only** (`packages/world-vercel/src/index.ts:40`); `world-local` and +`world-postgres` declare neither it nor `maxConcurrency`. + +Its predicate is narrower than the bug class, and the reason is its *shape*, +not the event type it watches. The marker advances on `hook_received` **or** +`step_completed`, but it is a **high-water mark** β€” the newest such write β€” and +the test is `stateUpdatedAt < marker`, strictly. So it detects a log truncated +at the end and is blind to a hole in the middle: when the withheld event is +*older* than one the reader can see, the reader's snapshot is never strictly +older than the mark. The hook direction is caught for the mirror-image reason β€” +the withheld `hook_received` is the newest out-of-band write and the +orchestrator's snapshot predates the sleep, so the fence fires and the run +reconciles. + +**`countGuard`** adds the count half: how many events the log holds at or below +`stateUpdatedAt`, compared against how many the caller loaded. It closes the +hole the watermark cannot see. It requires the caller to send +`stateEventCount` β€” and since #3145 (`1471f252f`) `@workflow/core` sends it on +every replay-context create, gated only by the `WORKFLOW_PRECONDITION_GUARD` +kill-switch, with workflow-server's own count guard defaulting on. So both +halves are armed together in production, and `countGuard` defaults here to +whatever the fence is set to. A run with the fence on and the count off is a +world that exists nowhere; the two scenarios that ask for it +(`step-vs-step-fork-fenced`, `in-flight-before-decision`) do so explicitly, +because isolating the watermark half is their whole subject. + +Where the runtime sends a count, the sim uses **that** value rather than its own +reconstruction (`loadedCount()`), so the guard is tested against the number the +real client computes; the reconstruction is the fallback for writes core does +not count. + +**Two ways the sim's guards are stronger than production's.** Both are +deliberate, and both mean a fenced green here is a claim about the *predicate*, +not about production's *deployment* of it: + +- The server's retained-id window is a FIFO in **insertion (commit) order** β€” + its Lua script prunes with `table.remove(ids, 1)`, oldest-inserted β€” while + `pruneRunEventIndex` here sorts by id and drops the smallest, i.e. mint order. + The two differ exactly when commits happen out of mint order, which is these + scenarios' whole subject, and they differ in *when* + `countRecordedAtOrBelow` goes indeterminate once a run passes 16 events. +- Production's watermark is best-effort: region-local Redis, failing open on + Redis errors, and blind to a webhook served in another region entirely (see + `outside-event-tracker.ts`'s own docs). The sim's is exact and in-process. + +**Overriding them for a whole run.** `RunScenarioOptions.preconditionGuard` +(`pnpm sim --fence` / `--no-fence`) forces the fence on or off for every +scenario, `undefined` leaving each spec to decide. Both halves move together: +the count guard is evaluated inside the same predicate, so disarming the fence +disarms it too. + +Forcing it *off* across the book asks whether anything relies on it. Violations +go **6 β†’ 8** mint-ordered, so it is load-bearing there; **0 β†’ 0** against an +append-only log, so it is dead weight once positions are assigned at commit. +This is a diagnostic rather than a world, and the number to read is the +violation count: a scenario whose subject is that the guard fired asserts that +with `sim.check` and fails by design when it does not +(`in-flight-before-decision-counted` today). + +### Fault injection + +**`withholdNextEvent(reads = 1)`** hides the next committed event from the +following N event-log reads. This is the only way a serial simulation can +produce "a write derived from an incomplete event load", the precondition a +real deployment reaches through concurrency. + +It is a faithful model rather than an approximation, because production reaches +the same ordering natively: `world-local` mints `evnt_${monotonicUlid()}` near +the top of `createImpl` (`packages/world-local/src/storage/events-storage.ts`) +and writes the file much later, so two concurrent creates take positions N and +N+1 and can land in the opposite order. Postgres does the same via `nextval` +before `COMMIT`. `world-local` defends this with `mintRunDominantEventKey` +(`src/storage/helpers.ts`) β€” but only for terminal run events; +`wait_completed` gets no re-derivation. + +One withheld read poisons a whole invocation, which is worth knowing when +reading a trace: after the next `step_completed` the runtime continues from its +cursor, fetching only events written strictly *after* that position. A withheld +event sitting before the cursor can never re-enter that invocation's view. +Incremental reads make the hole permanent. + +**`beginHookDelivery(token, payload)`** returns an `InFlightWrite` β€” a write +held between mint and commit, with `eventId` already fixed and `commit()` +still pending. Unlike a held writer, nothing is blocked meanwhile, because the +receiver is a separate process from the run's invocation. Holding an *inline* +write instead would stall the delivery that made it, and thus the reader too, +which is why the out-of-band writer is the one that can express this shape. + +### Changing the world instead of the runtime + +**`appendOnlyLog`** is the one option that alters the store's contract rather +than its strictness. With it on, an event takes its position in `append` instead +of at the handler boundary: a write that is still the newest when it commits +keeps the id it was already handed out under, and one that was overtaken while +it was held re-mints and takes the tail. + +That single move collapses both faults above into the same, weaker one. A hold +between mint and commit can no longer open a hole, because the held write is not +claiming a position while it waits β€” it has none until it lands. And +`withholdNextEvent` degrades from serving a read *around* the withheld event to +stopping it *at* the event, because a hole is not expressible in a log whose +order is its commit order. Both leave the reader short rather than wrong, and +short is precisely what the fence's watermark was designed to catch. + +Off by default: the sim exists to model the world that exists, and production +mints at the boundary because DynamoDB does not generate ids. The value of the +switch is differential β€” play the book both ways and the diff separates "fails +because of the mint-before-commit window" from "fails for some other reason". +No scenario in the book sets it; it is meant to be driven from +`RunScenarioOptions` or `pnpm sim --append-only`. + +--- + +## 6. The scenario surface + +### Spec + +```ts +interface ScenarioSpec { + id: string; // stable hyphenated handle; what `pnpm sim ` selects + name: string; // prose, expected to be reworded; the id is not + description?: string; + workflow: string | { workflowId: string }; // plain fn name, resolved via the build manifest + input?: unknown[]; + script?: ScenarioScript; // omitted = a control: run on the default schedule + selectNext?: SelectNext; // override queue delivery order + verifyReplay?: boolean; // default on for runs reaching completed/failed + expect?: { status?: ScenarioOutcome; output?: unknown }; // output: deep equality + limits?: ScenarioLimits; + preconditionGuard?: boolean; // advertise + enforce the optimistic-concurrency fence + countGuard?: boolean; // also enforce its count half + appendOnlyLog?: boolean; // position at commit, not at mint; see Β§5 +} +``` + +`RunScenarioOptions.appendOnlyLog` overrides the last of those for every +scenario in a run, which is how the whole book gets played both ways; +`undefined` there leaves each spec to decide. The mode a result was produced +under is recorded on `ScenarioResult.appendOnlyLog` rather than left to the +reader's memory. + +`expect.status` accepts the non-run outcomes (`stalled`, `budget-exceeded`) +because "this workflow deadlocks when the hook never arrives" is a property +worth pinning down rather than an accident to tolerate. + +There is deliberately **no way to expect a consistency violation.** A scenario +reproducing a corruption states the outcome the run *should* have had and fails +until the runtime delivers it. A red is an open bug, not a recorded +observation, and it goes green when the bug is fixed rather than when the bug is +seen once more. + +### Scripting + +`ScenarioApi` is the complete set of sanctioned external inputs β€” anything a +real deployment could do out-of-band has an entry, so the script is a complete +description of what happened: + +`deliverHook` Β· `beginHookDelivery` Β· `cancelRun` Β· `advanceTime` Β· +`withholdNextEvent` Β· `note` Β· `check` Β· `world` (read-only snapshot) Β· `runId` + +`Tempo` adds the steering: `writer` handles, plus the raw `park` / `until` / +`during` primitives. The vocabulary is borrowed from Python's `blanket`, which +does the same for `threading` primitives β€” the call *parks*, the script issues +the *permit*, and the resulting order of permits is the *tempo*. + +### Writers + +`sim.writer.orchestrator()` / `.step(shortName)` / `.anyStep()` / `.any()` +return a `Writer`. A handle is a **name, not a live object**: it can be taken +before the step exists and binds to whichever writer shows up under it. + +| method | phase | meaning | +|---|---|---| +| `runToEventProduced` | `before` | decided and submitted, nothing in the log | +| `runToEventCommitted` | `after` | durable, writer not yet resumed | +| `release` | β€” | let it go; idempotent | +| `isHeld` / `history` | β€” | inspection | + +Two implementation details of `release()` matter to scenario authors. It is +guarded by a `done` flag so double release is a no-op. And it awaits a full +macrotask turn before resolving β€” without that, `await release()` returns while +the resumed call is still queued as a microtask, and a scenario reading the log +on the next line sees the state it was trying to leave. + +### `runTo` is level-triggered + +It consults the history of points the writer has already reached *before* +arming anything, and throws `AlreadyPassedError` naming the call it happened at +if the point has gone by. + +The alternative β€” arm a watch and wait β€” is a hang. A held call blocks its +writer, and when that writer is the one the scheduler is inside, it blocks the +loop; so there is no quiescence to fall back on and no timer to eventually +fire. An edge-triggered wait on an edge that has passed is the one way to lose +this package's termination guarantee, so it is made impossible rather than +documented. + +Three consequences: + +- **Holds must be armed before they are needed.** To catch two writers at the + same point, start both waits and *then* await them. Awaiting the first before + starting the second yields the event loop, and the other writer may sail past. +- **`runTo` on an already-held writer releases it first**, and arms the new + watch *before* releasing. That order is load-bearing: the released writer can + reach the next point within the same turn β€” the `after` phase of the very + call it was held in is the common case β€” and a watch armed afterwards would + miss it. The same rule applies to authors sequencing two writers: arm B + before releasing A. +- **A call is two records, so `seq` cannot order them.** The `before` and + `after` phases of one call share a `seq`, so each recorded point carries its + own `ordinal` and the level check compares against that. + +A watermark tracks how far each writer has been advanced. Points at or before +it are "already consumed" and do not count as already-passed β€” asking twice for +`step_completed` means the *next* one, which is what the duplicate-delivery +scenarios need. + +### What is not offered + +Writers form a dependency graph β€” the orchestrator awaits its own step bodies β€” +so not every interleaving exists to be asked for, and an unsatisfiable `runTo` +can only be reported, not prevented. The runtime's await graph is not visible +from here, so true deadlock detection is out of reach; the substitute is a +per-`runTo` watchdog that reports where every writer was standing. + +--- + +## 7. Termination + +Every scenario terminates. Four budgets, layered so the most specific one +reports first: + +| budget | default | catches | +|---|---|---| +| `maxRunToWallMs` | 5 s | one `runTo` that will never be satisfied | +| `maxDeliveries` | 200 | a run that keeps re-enqueueing itself | +| `maxVirtualMs` | 365 d | `while (true) { await sleep('1d') }` | +| `maxWallMs` | 60 s | a genuinely non-terminating step body | + +`maxRunToWallMs` sits far below `maxWallMs` on purpose: it can name which +writer failed to reach which point and where the others were standing, and that +diagnosis is worth more than the generic "ran out of wall clock" the global +deadline can offer. It is clamped to `maxWallMs` so lowering the global budget +does not require remembering to lower this one. + +The scenario's global deadline must **not** be `unref`'d. An unref'd timer does +not hold the event loop open, so a total deadlock β€” every writer held, scheduler +blocked inside a held call, script awaiting the impossible β€” empties the loop +and exits Node with a bare "unsettled top-level await" instead of firing the +watchdog, which is precisely the case the watchdog exists for. The `finally` +already clears it, so it cannot outlive a scenario. + +Stream readers get the same treatment: a reader that parked on an unfinished +stream would deadlock the scenario, so readers park on a promise the *writer* +resolves and `abortOpenReaders()` releases any still parked at teardown β€” +turning a hang into a reported diagnostic. + +Outcomes are `WorkflowRunStatus | 'stalled' | 'budget-exceeded' | 'error'`. A +hook that never arrives is reported as a **stall naming the undelivered token**, +not a hang. + +--- + +## 8. Consistency checking + +Two independent checkers run over every scenario. + +### Invariants + +The store enforces most rules at write time by rejecting bad events β€” but "the +store rejected it" and "the log is actually consistent" are different claims, +and only the second is worth trusting. So `invariants.ts` re-derives everything +from the event log alone and compares against the entity rows. + +25 rules, grouped: + +``` +log.monotonic-order log.unique-event-id +run.created-first run.created-once run.terminal-is-last +run.entity-matches-log run.attributes-match-log run.output-materialized +run.resources-released +step.created-once step.started-after-created step.terminal-after-created +step.terminal-once step.no-restart-after-terminal +step.entity-has-log step.entity-matches-log step.attempt-matches-log +hook.token-unique hook.received-after-created +hook.dispose-once hook.no-receive-after-dispose +wait.created-once wait.completed-after-created +wait.completed-once wait.resume-at-stable +``` + +A violation is a bug somewhere β€” in the runtime that produced the sequence, in +the store that accepted it, or in the scenario that injected something +impossible. Which one is a question for the reader; the checker's job is only +to notice. + +### Replay verification + +The invariants check the log's *shape*. None of that answers the question +durability actually rests on: if a fresh process picked up this log tomorrow, +would it reconstruct the same run? + +The check is a **cold start with the answer withheld**. Take the finished log, +drop its terminal `run_*` event, load the rest into an empty world as durable +history, and deliver one queue message. The real runtime β€” the same +`workflowEntrypoint` a deployment serves β€” replays from the log and must +re-derive the event that was removed, with the same output. No step body +re-executes, since every `step_completed` is in the log and the step consumer +resolves from it, so anything the replay produces came from the log alone. + +In this frame, **replay is the serializability check.** A pass is pure, so +re-running it over the committed log asks whether the schedule had a serial +equivalent. Six failure ids: + +`replay.diverged` Β· `replay.suspended` Β· `replay.output-differs` Β· +`replay.log-differs` Β· `replay.status-differs` Β· `replay.budget` + +`replay.diverged` is the runtime raising `ReplayDivergenceError`, exhausting +its recovery replays, and failing the run with `CorruptedEventLogError`. +`replay.suspended` means the replay ran out of log before the workflow +finished β€” the log did not contain enough to rebuild the run. + +--- + +## 9. Current status + +Measured on branch `sim-world`. + +**Unit tests** β€” 72 passing across 8 files (`pnpm --filter @workflow/world-sim test`). + +**Scenarios** β€” `pnpm sim` in `workbench/sim-world`: + +``` +41 scenario(s): 35 passed, 6 failed, 6 consistency violation(s) +``` + +And the same book against an append-only log (`pnpm sim --append-only`): + +``` +41 scenario(s): 41 passed, 0 failed, 0 consistency violation(s) +``` + +Both numbers are the intended steady state; see "The six" below for which of +the six violations that second line closes on the merits and which close +because the correct answer itself changes. + +There was a seventh red until recently, `unclaimed-payload-under-fork`, and it +was a different animal: it tripped a `sim.check` rather than the replay +invariant, and it was red in *both* worlds, because nothing was wrong with its +log's positions β€” the runtime handed the workflow two resolutions in an order +the log did not record, so live and replay ran the same code, made the same +mistake, and agreed. Only the log disagreeing with itself caught it. #3406 +fixed the delivery-barrier ordering and it is now green in both worlds; the +scenario stays as that fix's regression test. + +With the fence forced off (`pnpm sim --no-fence`), violations go to **8** +mint-ordered and stay at **0** append-only β€” see Β§5. + +Replay verification across the book: **33 `ok`, 6 `MISMATCH`, 2 `skipped`** +(skipped where the run did not reach a terminal status). + +`run.ts` exits non-zero, and that is the intended steady state. The six +failures are reproductions of corruptions the runtime can still produce; each +states the outcome its own durable log implies and fails until the runtime gets +there, so the failure line names both sides (`expected "afterSlow:doc-26", got +"afterFast:doc-26"`). + +**The number is the thing to watch: six today.** A seventh is a regression; +five means something got fixed and a scenario is ready to retire. + +That makes the book a poor plain CI gate, which is what `--report-only` is for: +it prints every failure and exits 0, so a job can *publish* the book's current +state rather than block on it. `--summary-file` writes one collapsed +`
` β€” a visible line carrying the count and a green or orange dot, the +whole table behind it β€” for a PR comment or `$GITHUB_STEP_SUMMARY`, and +`--detail-file` writes the full colour-free trace as an artifact to read when a +number moves. Deliberately nothing above the fold but the count: six are red on +purpose, so a comment that leads with the failures leads with the part that is +not news, and grows a wall of text on exactly the PRs that changed nothing. The +workbench's `pnpm test` is `--report-only --summary-file`, so a recursive +`pnpm -r test` stays green and still says what happened; `pnpm sim` stays +strict, so running it by hand fails loudly. + +### The six + +The `fix` column names the specific change that closes the scenario. `shown +green by` is the stronger claim: a *passing* scenario that is this one with that +fix armed, same workflow and same tempo, one flag apart. Where it says "none +yet", the fix is identified by argument but nothing in the book proves it. + +| scenario | mechanism | fix | shown green by | +|---|---|---|---| +| `stale-read-step-count-fork` (doc-23) | `withholdNextEvent(1)` + `deliverHook`; hook at `#7`, `wait_completed` at `#8`, no-hook branch at `#9` | `preconditionGuard` β€” the withheld `hook_received` is the newest out-of-band write and the orchestrator's snapshot predates the sleep, so the watermark fires | `stale-read-step-count-fork-fenced` (doc-24) | +| `stale-read-equal-step-counts` (doc-25) | same fault on a fork whose branches emit one step each | `preconditionGuard`, for the same reason | none yet | +| `step-vs-step-fork` (doc-26) | two of the run's own `step_completed` events, one delivery | `countGuard`. **Not** `preconditionGuard`: the withheld completion is a hole in the middle of the log, which moves no high-water mark (Β§5) | none yet | +| `step-vs-step-fork-fenced` (doc-27) | same, `preconditionGuard: true`, zero rejections | `countGuard`. This row *is* the proof that the watermark half does not fix doc-26 | none yet | +| `in-flight-before-decision` (doc-29) | `beginHookDelivery`, committed before the decision is written | `countGuard` | `in-flight-before-decision-counted` (doc-30) | +| `in-flight-after-decision` (doc-31) | `beginHookDelivery`, committed after the decision | none in the SDK. Needs an append-tail fence β€” `assertSlotAboveTail`, `vercel/workflow-server#692` | β€” | + +Those handles are `ScenarioSpec.id`, and they select: `pnpm sim +in-flight-after-decision` plays one row of this table. + +So: **two** of the six have their fix demonstrated by a paired green scenario, +**three** have a fix identified but unproven here, and **one** has no fix at all. +Writing the three missing pairs is the obvious next increment. + +Note what the `fix` column does *not* mean. `countGuard` closing doc-29 is a +statement about the World implementation *and* about production's predicate: +core has sent `stateEventCount` on every replay-context create since #3145 and +the server's count guard defaults on (Β§5). What it is not is a statement about +production's *deployment* of that predicate, which is region-local, fails open, +and prunes its window in a different order than this store does β€” all three +noted in Β§5. So the honest reading of the column is: four of the six have a fix +whose predicate is armed in production today, and whether it fires there depends +on conditions the sim does not model. + +**The append-only log closes all six, in two different senses β€” and the split +is four and two, not three and three.** Four (doc-23, doc-25, doc-26, doc-27) +close on the merits, with the book asking them exactly what it asked before: the +reordering was the fault, and once positions are assigned at commit the withheld +read degrades from a hole to a truncation, which the fence can see. The +remaining two (doc-29, doc-31) close because the branch the run ends on changes. +A hook that commits after the timeout genuinely *is* after it when the tail is +the only place a write can land, so the log records the timeout first and the +run that settled is the run the log describes. + +**No expectation is restated per world, and there is no mechanism to.** The +first cut of this had one β€” an `expectAppendOnly` field on three scenarios, +naming a second correct output. It was the wrong instrument, for a reason worth +keeping written down. A scenario is one sequence of advances. The only thing a +world changes is what a read returns. The branch a run ends on is decided by +what it read, so pinning the branch pins a consequence of the world rather than +a property of the run, and any expectation that then has to be restated per +world is evidence the pin was wrong β€” not evidence that a second answer is +needed. The three now assert what holds in both worlds (the run completes) and +report the branch in the trace. + +That costs nothing, because the expectations were never what caught the fault. +The load-bearing assertion is the invariant: **the log a run wrote must be a log +the runtime can replay back into that same run**. It is world-independent, on by +default (`verifyReplay`), and it is what all six reds trip. Measured, not +assumed: strip every `expect` in the book and the violation counts do not move +β€” 6 mint-ordered, 0 append-only, the same six by name. (Pass/fail does move by +one, and only for a bookkeeping reason: `hook-never-arrives` expects `stalled`, +and a stall's reason is reported as a problem unless the scenario said it was +expecting one.) That also removes +the one place where the flag's scoreboard rested on a judgement about what the +right answer *is* rather than on something the harness checks on its own. + +doc-30 is worth a line because it was the third `expectAppendOnly` and is *not* +one of the six β€” mint-ordered it already passes, since `countGuard` catches +there what the watermark half misses. Its branch moves under the flag for the +same reason its uncounted twin's does, so the old pinned output would have +turned a green scenario red. What made it distinct from doc-29 was never the +branch anyway; it is that the count half of the fence fires at all. That is now +asserted directly, matched on the guard's own message and true in both worlds β€” +and it fails if `countGuard` is turned off, which is the check that a bare +`rejections().length > 0` would have missed, since doc-29 rejects too. + +Two details worth keeping: + +- Hook delivery participates. `beginHookDelivery` still reserves a position at + the handler boundary; under the flag the reservation stops being binding and + the write re-mints at the tail if anything overtook it (`positionAtCommit`). +- doc-30's 412 still fires, and now saves nothing. The count guard counts events + at or below the caller's watermark, the watermark is a millisecond, and a hook + committing after the timeout within the same virtual millisecond is still "at + or below" it. The reload finds nothing to correct and the run settles anyway. + That false positive is the standing cost of the count half of the fence once + the log is append-only, and doc-30's trace is where to see it. + +Four of the six are hook-driven and two deliberately are not β€” the pair proves +the corruption needs no out-of-band event type. All the pure hook-timing +scenarios pass: placing a hook precisely is what works. What fails is a hook +that is durable in the log but absent from the read the live pass decided on. + +The last row is the only one with no fix in the SDK: the hole opens *after* the +write that should have fenced it, in the quiescent gap between deliveries where +the run makes no writes and so meets no checks. `assertSlotAboveTail` in +`vercel/workflow-server#692` is the append-tail fence for it. + +--- + +## 10. Limits + +**Concurrent invocations are out of reach.** The scheduler does +`await deliver(...)`, so two flow deliveries for one run cannot overlap. +Reaching that would need concurrent delivery with hold points to pin the +interleaving. The gap matters because it is a real production route: +`resumeHook` writes `hook_received` *and* enqueues a flow message, so two +deliveries end up in flight β€” one writing `wait_completed` and deciding +no-hook, one seeing the hook and deciding hook-branch β€” racing to create the +same ordinal, with every reader holding a perfectly consistent view. Just +different ones. + +Two step bodies inside one delivery are genuinely concurrent and separately +steerable, which is enough to reach the interesting corruption without a second +invocation. That is why the limit has been acceptable so far. + +**The parallel hook-resume path is never exercised.** `resumeHook` picks +parallel ("lazy") vs sequential from `world.capabilities.hookResumeDedup` (or a +fresh server attestation). `world-local` declares it and `world-vercel` attests +it per lookup, so **every real world takes the parallel path**, where the queue +publish races the `hook_received` write and the consumer re-ensures the event +through the durable `(runId, resumeId)` claim. The sim advertises neither the +capability nor a `resumeId` dedupe, so every sim hook delivery takes the +sequential path β€” meaning the hook-timing shapes in this book are the *legacy* +shape, not the one production runs. Closing this needs `(runId, resumeId)` +dedupe in the store plus the capability; it is the largest single gap for a +package about hook races. + +**Also untested:** turbo / optimistic-inline-start, which skip replays and so +give a stale branch somewhere to hide; and the fence's same-millisecond +behaviour, where an equal `stateUpdatedAt` passes by design as anti-livelock. + +**Not modelled at all:** the concurrency machinery `world-local` needs and this +store omits β€” claim files, per-entity locks, staged/promoted hook events, +canonical event-id pinning after a crash. Bugs in those are invisible here. + +--- + +## 11. A caveat worth stating + +A simulated world only produces trustworthy results while its model matches +reality. Every simplification in Β§5 and every limit in Β§10 is a place where a +green scenario could be green for the wrong reason. The mitigations are that the +store keeps every *rejection* the real one performs, that the runtime under test +is the real `workflowEntrypoint` running real compiled workflow code, and that +every scenario ends by replaying its own log through that same runtime β€” but +none of those is a proof, and a red here is worth more than a green. diff --git a/packages/world-sim/README.md b/packages/world-sim/README.md new file mode 100644 index 0000000000..6646c36578 --- /dev/null +++ b/packages/world-sim/README.md @@ -0,0 +1,568 @@ +# `@workflow/world-sim` + +A deterministic, fully in-memory World for **playing out workflow scenarios** +and **checking that the world contract holds**. + +It exists to answer questions that a real World cannot be asked, because in a +real World they are races: + +> What happens if the approval webhook arrives *after* `step_started` is +> durable but *before* the workflow gets control back? + +In `@workflow/world-local` you would answer that by polling in a loop and +hoping. Here you state it, and it is what happens β€” every time, byte for byte. + + + +```ts +const wf = sim.writer.orchestrator(); +await wf.runToEventCommitted('step_started', 'reserveInventory'); +await sim.deliverHook('approval:doc-1', { approved: true }); +await wf.release(); +``` + +The resulting event stream: + +``` + 0 +0ms wf run_created approvalWorkflow input=<17B> + 1 +0ms wf run_started + 2 +0ms wf hook_created hook_…KX token="approval:doc-1" + 3 +0ms wf step_created step_…KY reserveInventory input=<58B> + 4 +0ms wf step_started step_…KY reserveInventory + +0ms wf >> held "orchestrator -> step_started step=reserveInventory (committed)" at events.create:after + 5 +0ms ext hook_received hook_…KX token="approval:doc-1" payload=<44B> + 6 +0ms reserveInventory step_completed step_…KY reserveInventory result=<22B> + … +``` + +The second column names the writer. The indented `hook_received` is written by +the scenario (`ext`) from *inside* the `events.create` call that committed +`step_started`, while the orchestrator is held in it. Advance a different +writer instead β€” `sim.writer.step('reserveInventory')` β€” and the same workflow, +same input and same output produce a different log, which is the point. + +--- + +For how it is built β€” the interception model, the store's guards, the +determinism machinery, and the test status β€” see [DESIGN.md](./DESIGN.md). + +--- + +## The model + +Three rules, and everything else follows from them. + +**1. The World API is the schedule.** Every method is wrapped so a scenario can +run code `before` a call starts, or `after` its effect is committed but before +the awaiting caller is resumed. Since the World API is the only channel between +the runtime and the outside, that is a complete set of injection points. + +**2. Nothing happens on its own.** `queue()` records a message and returns; it +never dispatches. The scheduler picks the next message β€” always the minimum by +`(readyAt, enqueueSeq)` β€” hands it to the flow handler, and waits for it to +finish before looking again. One delivery is in flight at a time. + +**3. Time is a number the scheduler assigns.** `sleep('30d')` becomes a queue +message dated 30 days out; delivering it means moving the clock, not waiting. +`Date.now()` and `new Date()` read the virtual clock while a scenario runs +(timers are left alone β€” the runtime uses zero-delay macrotasks as ordering +barriers, and faking those would change the interleavings we came to observe). + +Consequence: **scenarios terminate**. A month-long sleep costs microseconds. A +hook nobody delivers drains the queue and is reported as a *stall*, naming the +token that was never sent, instead of hanging. Delivery count, virtual span and +wall time are all capped as a backstop. + +## Consistency checking + +Every scenario ends with the event log re-read and the entity state +re-derived from it. `checkInvariants` verifies, among others: + +| Rule | What it means | +| --- | --- | +| `log.monotonic-order` | Append order equals `(createdAt, eventId)` sort order β€” replay sees what happened | +| `run.created-first`, `run.created-once`, `run.terminal-is-last` | Run lifecycle shape (a step already running may still close out after termination) | +| `step.no-restart-after-terminal`, `step.terminal-once` | A finished step stays finished | +| `step.entity-matches-log`, `step.attempt-matches-log`, `run.entity-matches-log` | Materialized rows are a pure fold of the log | +| `hook.token-unique`, `hook.no-receive-after-dispose` | One live hook per token; disposal is final | +| `wait.resume-at-stable`, `wait.completed-once` | A wait's deadline is not rewritten (the sleep consumer treats a change as replay divergence) | + +### Replay verification + +Shape checks say the log is well formed, not that it is *enough* to rebuild the +run, so every scenario reaching `completed` or `failed` ends with a cold start: + +1. Take the committed log and withhold its terminal `run_*` event. +2. Seed the rest into an empty world as durable history. +3. Deliver one queue message to the same `workflowEntrypoint` a deployment + serves, with the clock pinned to the instant the run ended. +4. The runtime must replay from the log alone and re-derive the event that was + withheld, with the same output. + +No step body re-executes: every `step_completed` is in the log, so anything the +replay produces came from the log and nothing else. Failures are named: + +| Rule | What happened | +| --- | --- | +| `replay.diverged` | The runtime could not follow its own history: `REPLAY_DIVERGENCE` / `CORRUPTED_EVENT_LOG` | +| `replay.suspended` | The replay ran out of log before the workflow finished | +| `replay.output-differs`, `replay.status-differs` | It finished, with a different answer | +| `replay.log-differs` | It re-derived a different tail than the one withheld | + +Skipped for `cancelled` and `stalled` runs: their terminal event came from an +operator, or never existed, so there is no workflow-derived answer to reproduce. + +The store behind all of this is a compact reference implementation of the same +event β†’ entity state machine `@workflow/world-local` runs on the filesystem. Its +cross-process race machinery (claim files, per-entity locks, staged hook events, +canonical event-id pinning) is dropped, since a scenario is single-threaded; +every *validation* is kept, because rejections are the observable contract. + +## World behaviors + +A scenario picks the world it plays in, and each behavior below changes a rule +the runtime is written against. + +**Mint-ordered log** β€” the default. A position is assigned when the event's +handler mints its id, and the event is committed to storage separately. The id +*is* the log's sort key, so a write held between the two lands *behind* events +minted later and committed sooner: an event can arrive in the past, and a read +taken in between saw a log the log itself went on to contradict. + +**Append-only log** (`appendOnlyLog: true`) β€” a position is assigned at commit. +A write overtaken while it was held gives up its position and re-takes the tail. +Two things follow: + +- Log order is commit order. Nothing is inserted behind a row a reader has + already seen, so no two reads can disagree about the past. +- Every read is a *prefix* of the log. A read can be short β€” missing a write + that has not committed yet β€” but never self-inconsistent. Staleness collapses + into lag, and lag is what an optimistic-concurrency fence can see; a hole is + what it cannot. + +Uncontended writes are untouched either way: a position that is still the newest +when it commits keeps its id, so a scenario that never holds a write mid-flight +produces a byte-identical log in both. `withholdNextEvent` follows the same +rule β€” a hole in the mint-ordered log, a truncated tail under append-only β€” +which is why `StaleRead` reports `{ eventId, hidden, truncated }` and the trace +distinguishes a lagging read from a stale one. + +**Precondition fence** (`preconditionGuard: true`) β€” rejects a write whose +`stateUpdatedAt` snapshot is strictly older than the newest externally +originated event. It is a high-water mark, so it sees a log truncated at the end +and is blind to a hole in the middle. + +**Count guard** (`countGuard: true`) β€” adds the other half: how many events the +log holds at or below `stateUpdatedAt`, against how many the caller loaded. It +closes the hole a watermark cannot see, and requires the caller to send +`stateEventCount`. It is evaluated inside the fence's predicate, so it is only +live when the fence is. + +Each is a spec field, and `RunScenarioOptions` carries a run-wide override β€” +`pnpm sim --append-only`, `--fence` / `--no-fence` β€” where `undefined` leaves +each scenario's own choice alone. Playing one book under two behaviors and +diffing the results is what the pair is for; [DESIGN.md +Β§5](./DESIGN.md#the-two-guards) has the guards in full. + +## Usage + +You write two things: a **workflow**, and a **script** that controls how that +workflow executes. + +The workflow is ordinary workflow code, compiled the way a deployment compiles +it: + +```ts +// workflows/index.ts +async function stepA(input: string) { + 'use step'; + return `a:${input}`; +} + +async function stepB(input: string) { + 'use step'; + return `b:${input}`; +} + +export async function twoStepsWorkflow(input: string) { + 'use workflow'; + const [a, b] = await Promise.all([stepA(input), stepB(input)]); + return `${a}|${b}`; +} +``` + +Both steps are in flight at once, so which of them reaches the log first is a +race. The script decides it: hold `stepA` before its completion is assigned a +position, let `stepB` commit, then let both go. + +```ts +import type { ScenarioSpec } from '@workflow/world-sim'; + +const spec: ScenarioSpec = { + // The stable handle: what a bug report cites and `pnpm sim ` selects. + // The prose `name` beside it is free to be reworded. + id: 'b-lands-first', + name: 'stepB lands in the log before stepA', + // Named from the build manifest β€” no client transform needed. + workflow: 'twoStepsWorkflow', + input: ['x'], + script: async (sim) => { + const a = sim.writer.step('stepA'); + const b = sim.writer.step('stepB'); + + // Calling an advance starts watching for its point; awaiting it waits for + // the writer to get there. Start both watches, then await both β€” asking + // for a point that has already gone by is an error, not a wait. + const watchA = a.runToEventProduced('step_completed'); + const watchB = b.runToEventCommitted('step_completed'); + await watchA; + await watchB; + + await b.release(); + await a.release(); + }, + expect: { status: 'completed', output: 'a:x|b:x' }, +}; +``` + +`stepA` is held before it takes a position, so `stepB` gets the earlier one β€” +`#6 stepB`, `#7 stepA` β€” on every run, in either order the runtime would +otherwise have picked. + +Playing it needs the compiled bundle, because the orchestrator runs from a code +string inside a VM: + +```ts +import { + loadFlowHandler, + renderScenario, + runScenario, + type ScenarioSpec, +} from '@workflow/world-sim'; +// Separate entry on purpose: this one reaches SWC and esbuild through +// `@workflow/builders`, and playing a scenario should not drag a compiler into +// the module graph. +import { buildSimBundle } from '@workflow/world-sim/build'; + +declare const spec: ScenarioSpec; // the one above + +const bundle = await buildSimBundle({ cwd: process.cwd(), dirs: ['workflows'] }); +const handler = await loadFlowHandler(bundle.flowBundlePath); + +const result = await runScenario(spec, { + handler, + workflowIds: bundle.workflowIds, +}); +console.log(renderScenario(result)); +``` + +`expect` states what *correct* looks like, which is not always what the runtime +does. There is deliberately no way to expect a consistency violation: a scenario +reproducing a corruption declares the outcome the run should have reached and +stays red until the runtime delivers it, because a suite that goes green by +recording the bug gives no signal on the day someone fixes it. + +[`workbench/sim-world`](../../workbench/sim-world/README.md) is the worked +example β€” a book of scenarios, a CLI that plays them, and a guide to adding +one. + +## Reading the output + +Events are referred to one way and one way only: **by log position**, so a claim +about the output is one a reader can check against it. + +`#12` is the twelfth event in the log sorted the way `events.list` sorts it, +`(createdAt, eventId)`; `@7` is the resource created at position 7. Ids in +violation messages are rewritten to positions on the way out. + +The trace prints in **commit** order and is numbered in **log** order, so a run +whose log disagrees with the order its writers committed in shows up as +positions counting backwards: + +``` +# 8 +1.0m wf wait_completed @6 +# 7 +1.0m ext hook_received @2 token="count:doc-29" +# 9 +1.0m wf step_created @9 settle +``` + +The hook owns position 7, the timeout at 8 was committed first, and the branch +at 9 went with the timeout. Out-of-order positions are highlighted when colour +is on. + +Colour is applied only when stdout is a terminal, and is off under `NO_COLOR` +or `--no-color`; pass `{ color: true }` to force it. With colour off the output +is plain ASCII, stable enough to check in as a golden file. + +## API reference + +Three things a script works with. A **writer** is a thread of execution. An +**advance** moves one writer to a named place and holds it there. A +**withholding** hides something from readers without holding anyone. + +### Writers + +A run is not one program: several writers append to one event log, and each +write crosses the world boundary, is assigned a position in the event log, and +is committed to storage. + +| writer | handle | what it is | what it writes | +| --- | --- | --- | --- | +| `orchestrator` | `sim.writer.orchestrator()` | The workflow function and the runtime around it, committing at a suspension point. One per queue delivery. | the run lifecycle, `step_created` / `step_started`, `hook_created`, `wait_*` | +| `step:` | `sim.writer.step('')` | One step body, running inline with full Node access. Two steps sharing a function name share the writer. | its own `step_completed` / `step_failed` / `step_retrying`, and any `attr_set` from step context | +| `external` | none β€” see [Withholdings](#withholdings) | The scenario, acting as a webhook receiver or an operator | `hook_received`, `run_cancelled` | + +Two step bodies in a *single* delivery are already two writers racing to the +same log: no second invocation and no real threads are required. That is why the +vocabulary is per-writer rather than per-invocation. + +`sim.writer.anyStep()` and `sim.writer.any()` are handles that match more than +one writer β€” whichever reaches the advance first. A handle is a *name*, not a +live object, so `sim.writer.step('slow')` can be taken before that step exists. +`sim.writer.seen()` lists the ids observed so far, in first-appearance order. + +### Advances + +An advance tells one writer to move to a named place and hold there until +`release()`. Every other writer keeps running, so whatever the script does in +between is guaranteed to land first. + +**Calling an advance starts watching; awaiting it waits for the hold.** The two +are separate on purpose: `const p = wf.runToEventCommitted(…)` is already +watching for that point, and `await p` only blocks the script until the writer +gets there. So a script that needs two writers held at once starts both +watches, then awaits both. + +```ts +import type { ScenarioScript } from '@workflow/world-sim'; + +const script: ScenarioScript = async (sim) => { + const wf = sim.writer.orchestrator(); + const reserve = sim.writer.step('reserveInventory'); + + // Hold just after step_started is committed and before the orchestrator is + // resumed β€” the window the whole instrument exists for. + await wf.runToEventCommitted('step_started', 'reserveInventory'); + sim.check('no payload yet', !sim.world.events().some((e) => e.eventType === 'hook_received')); + await sim.deliverHook('approval:doc-1', { approved: true }); + + // Start the next watch BEFORE releasing: a released writer can reach the + // next point within the same turn, and a watch started afterwards has + // missed it. + const done = reserve.runToEventCommitted('step_completed'); + await wf.release(); + await done; + await reserve.release(); +}; +``` + +| method | writer | description | +| --- | --- | --- | +| `wf.runToEventProduced(type, opts?)` | any | Hold once the event has crossed the world boundary β€” formed, attributed, in the trace β€” and before it is assigned a position in the event log. Anything committed to storage during the hold sorts *ahead* of it. | +| `wf.runToEventCommitted(type, opts?)` | any | Hold once the event is committed to storage, before the writer resumes. | +| `wf.release()` | the held one | Let the writer go. Idempotent; awaiting it yields the event loop, so the writer has really moved by the time it resolves. | +| `wf.isHeld()` / `wf.history()` | β€” | Is it held / where it has been. | +| `sim.park(match, label?)` | whichever matches | Hold the next matching call, whoever makes it. | +| `sim.until(match, label?)` | whichever matches | Wait for a matching call, without holding it. | +| `sim.during(match, body)` | whichever matches | `park`, run `body` while it is held, then release. | + +`type` is one event type or several. `opts` is a step name as a bare string, or +`{stepName, token, correlationId, where, label, timeoutMs}`. + +Both advances hold a writer whose event has no position yet, so a write that +commits during the hold sorts *ahead* of it. For the other order β€” an event that +already owns an earlier slot and has not appeared β€” hold the write itself with +[`sim.beginHookDelivery`](#withholdings), which reserves the position and hands +back a `commit()`. Under `appendOnlyLog` that reservation is provisional: an +overtaken write gives it up and re-takes the tail, which is exactly how the world +closes the gap. See [World behaviors](#world-behaviors). + +`runTo` is **level-triggered**: it consults recorded history, so a point this +writer already passed is an `AlreadyPassedError` naming the point rather than a +wait that never ends. Asking twice means "the next one". Each advance carries a +watchdog (`limits.maxRunToWallMs`) whose timeout reports where *every* writer +was standing, which is a diagnosis rather than the scenario's global budget +running out. + +Two mistakes are worth knowing, and the errors name both: + +- **Watching too late.** Releasing writer A before B's watch has started. B's + step body may already be in flight and commit during the release. +- **Naming the wrong writer.** `step_started` is the orchestrator's write; + `step_completed` is the step body's. The wrong one is a timeout. + +`park` / `until` / `during` take a raw match object and are what the writer +handles are built from. Fields are ANDed; `eventType` implies `events.create`, +`stepName` accepts the machine name or the plain function name, `where` covers +what the declarative fields cannot say, and `phase` defaults to `'after'`: + +``` +{ call: 'events.create' | 'queue' | 'runs.get' | … , phase: 'before' | 'after', + eventType, stepName, correlationId, token, runId, writer, failed, where } +``` + +Reach for them when the point is a *state* rather than a name. `where` is the +one thing a level-triggered `runTo` cannot re-check against history, so a +`where` wait is edge-triggered and leans on its timeout. + +The park/permit model β€” and the word *tempo* for the resulting order β€” is lifted +from [`blanket`](https://bernat.tech/posts/blanket-deterministic-threading/), +which does this for Python's `threading` primitives. The mapping is direct: a +world call is a transaction, the `after` phase is its parking state, and +`release()` is the permit. + +A script is the only way to hang this simulator, because a held call blocks its +writer and in the limit the scheduler. Three guards close that: the per-advance +watchdog above, the runner reporting what a script was still waiting for instead +of awaiting it forever, and a wall-clock deadline that releases every held call +and rejects every pending wait. A script that throws is reported as a scenario +problem rather than a World error, so a broken script is never misread as a +runtime bug. + +### Withholdings + +A withholding hides something from readers without holding the writer that +produced it. An advance stops one thread; a withholding lets every thread run +and changes what storage answers. + +| method | writer | description | +| --- | --- | --- | +| `sim.withholdNextEvent(reads?)` | whichever commits next | Hide the next event committed to storage from the next `reads` event-log reads (default 1). Call it immediately before the write to hide. | +| `sim.beginHookDelivery(token, payload)` | `external` | Deliver a hook, withheld between its two halves: assigned a position in the event log, not committed to storage. Returns `{eventId, commit()}`. | + +`beginHookDelivery` is the one place inside an `external` writer a script can +reach, and it is a withholding rather than an advance because holding that +writer would be the wrong model: an out-of-band receiver is a separate process, +so nothing of the run's is blocked while its write is in flight. Holding an +inline write would stall the delivery that made it, and the reader with it. + +Both change shape with the log. Under `appendOnlyLog` a withheld read is cut +short at the withheld event instead of missing it from the middle β€” the log can +be behind, never wrong β€” and an overtaken hook re-takes the tail on `commit()`. + +### Everything else a script can do + +| | | +| --- | --- | +| `deliverHook(token, payload)` | Runs the real `resumeHook()` β€” the same code an out-of-band webhook receiver would | +| `cancelRun(reason?)` | Cancel the run under test | +| `advanceTime(ms)` | Jump the virtual clock | +| `deliverQueued(select?)` | Deliver one queued message now, concurrently with a held writer | +| `note(msg)` / `check(name, cond)` | Record a marker / an assertion in the trace; a false check fails the scenario | +| `world` | Read-only snapshot: runs, events, steps, hooks, waits, pending messages, rejected calls | +| `appendOnlyLog` | Which log this run is playing against β€” for *phrasing* a check, never for branching the tempo | + +A scenario with no script at all is a control: the run plays out on the default +schedule, and the only question is whether the log it leaves reproduces it. + +#### `deliverQueued`, and why it is not an advance + +The scheduler is strictly serial: one message at a time, and the clock only +moves when it picks the next one up. So a held writer freezes virtual time +along with everything else, and a whole family of interleavings is simply +unreachable from the advances above β€” anything of the form *a timer fires while +a step result is outstanding*. Both halves need to be in flight at once, and the +loop will only ever have one. + +`deliverQueued` takes a message out of the pending set and delivers it right +there in the script, so it runs alongside the held writer rather than after it. +`takeById` removes it first, so the loop can never pick up the same message: the +two are different deliveries running concurrently, not a race for one. + +That concurrency is real, and so is its fallout. Two flow deliveries for one run +will collide the way they do in production β€” expect `EntityConflictError` and +`HookNotFoundError` in the rejection list once both branches finish. Those are +the deliveries losing races they are supposed to lose, not violations. + +The default picks `pending[0]`, matching the loop's own order. Usually you want +to choose: a hook delivery enqueues a flow message of its own and it sorts +earlier than the timer you are almost certainly after. + +```ts +import type { Tempo } from '@workflow/world-sim'; + +declare const sim: Tempo; // the `script` parameter + +const fired = sim.deliverQueued( + (pending) => pending.find((m) => m.readyAtMs > sim.world.nowMs())?.messageId +); +``` + +Note the missing `await` β€” awaiting it here would wait for the delivery to +*finish*, which defeats the purpose. Arm a hold on the writer that delivery will +wake, fire it, await the hold, and the two are now interleaved. Await the +returned promise at the end to assert it found something. + +## Extending the simulator + +Changing the instrument itself, routed by task β€” adding a *scenario* needs none +of it, and is +[`workbench/sim-world/README.md`](../../workbench/sim-world/README.md#adding-a-scenario). +The module map is [DESIGN.md Β§1](./DESIGN.md#1-module-map). + +| I want to… | Change | Read first | +| --- | --- | --- | +| let scripts hold at a point the API can't name | `world.ts` β€” the call-point wrapper, and `CallMatch` in `types.ts` | [Β§3 Interception](./DESIGN.md#3-interception) | +| add a phase to an existing call | `CallPhase` in `types.ts`, where `world.ts` parks on it, plus the writer op that names it | [Β§3 Two phases](./DESIGN.md#two-phases-and-a-third-hold-that-is-not-one) | +| add a rule the log must satisfy | `invariants.ts`, plus the rule table above | [Β§8 Consistency checking](./DESIGN.md#8-consistency-checking) | +| add or change a writer kind | `writers.ts` for the handles, `world.ts` for attribution | [Β§3 Writer attribution](./DESIGN.md#writer-attribution-is-derived-not-instrumented) | +| add a fault injector | `store.ts` β€” next to `withholdNextEvent` and the guards | [Β§5 Fault injection](./DESIGN.md#fault-injection) | +| change what a read returns | `store.ts` `applyWithhold` | [Β§5 The store](./DESIGN.md#5-the-store) | +| change where an event lands | `store.ts` `positionAtCommit` / `mintEvent` | [World behaviors](#world-behaviors) above | +| add a spec field | `ScenarioSpec` in `scenario.ts`, `RunScenarioOptions` beside it, then `run.ts` for the CLI flag | [Β§6 Spec](./DESIGN.md#spec) | +| change the replay check | `replay.ts` | [Β§8 Replay verification](./DESIGN.md#replay-verification) | +| change the output | `report.ts` β€” `renderScenario`, `renderSummary`, `renderMarkdownSummary` | [Reading the output](#reading-the-output) above | + +Four things worth knowing before you start: + +**The package entry is the scenario surface, not the whole package.** +`index.ts` exports what it takes to write a scenario, play it and render the +result. The construction kit β€” `createSimWorld`, `createSimStore`, `driveQueue`, +`verifyReplay`, `checkInvariants`, the clock β€” is imported from its own module, +so adding an option to one of them is not a change to the package's public +signature. Promote a name to the entry when something outside the package needs +it, not before. + +**A new world flag is tri-state at the runner.** `ScenarioSpec` carries the +scenario's own choice, `RunScenarioOptions` the run-wide override, and +`undefined` means "leave it to the spec" β€” not the same as `false`, because a +scenario that asked for the flag must keep it. `run.ts` maps `--x` / `--no-x` +onto that, and the resolved value reaches `createSimWorld` and the chips line. + +**Anything a scenario can observe has to survive replay.** `verifyReplay` +re-plays the log in a fresh world built from the same options, so a store rule +that is not applied there turns every scenario using it red for the wrong +reason. + +**Tests come in two shapes.** `src/*.test.ts` are vitest units against the +pieces in isolation β€” copy `store.test.ts` for anything that changes what the +log looks like, where the append-only block is written as pairs asserting +*opposite* outcomes in the two worlds. The scenario book is the integration +test; run it before and after and diff the counts. + +## What this does *not* give you + +Worth being explicit, because the guarantees are narrower than "deterministic": + +- **Determinism is world-level.** Step bodies are ordinary Node code. A step + that calls `Math.random()`, reads a file, or hits the network is as + nondeterministic here as anywhere. Keep step bodies pure, or stub them. +- **Only one interleaving per scenario.** Deliveries are serialized, so a + scenario pins *one* schedule rather than searching the space of them + (`selectNext` picks which queued message goes next). Same trade `blanket` + makes: it *reproduces* orderings you can describe, it does not *discover* + ones you can't. +- **The store is a reimplementation, not the real thing.** It models + `world-local`'s semantics rather than delegating to them, so it could in + principle agree with the runtime while a real world disagrees. The fix is + conformance testing: make the storage layer pluggable, play the same book + against `world-local`, and diff the event streams. +- **"Before the workflow resumes" is about the log, not the CPU.** The hook is + committed before the intercepted call returns, so it is in the log before the + runtime's next read of it. Whether the runtime *observes* it on the next + replay depends on optimizations that can skip a re-read β€” visible in the + trace. +- **One scenario at a time per process.** The virtual clock and the World are + process-global singletons. +- **Not a deployable World.** It has no persistence and no concurrency; it is a + test instrument, and is intentionally not listed in `worlds-manifest.json`. diff --git a/packages/world-sim/package.json b/packages/world-sim/package.json new file mode 100644 index 0000000000..dd57fa8eb9 --- /dev/null +++ b/packages/world-sim/package.json @@ -0,0 +1,48 @@ +{ + "name": "@workflow/world-sim", + "version": "0.0.0", + "private": true, + "description": "Deterministic in-memory simulation World for Workflow SDK consistency checks", + "type": "module", + "main": "dist/index.js", + "types": "dist/index.d.ts", + "files": [ + "dist" + ], + "license": "Apache-2.0", + "repository": { + "type": "git", + "url": "https://github.com/vercel/workflow.git", + "directory": "packages/world-sim" + }, + "exports": { + ".": { + "types": "./dist/index.d.ts", + "default": "./dist/index.js" + }, + "./build": { + "types": "./dist/build.d.ts", + "default": "./dist/build.js" + } + }, + "scripts": { + "build": "tsc", + "dev": "tsc --watch", + "clean": "tsc --build --clean && rm -rf dist", + "test": "vitest run src", + "typecheck": "tsc --noEmit" + }, + "dependencies": { + "@workflow/builders": "workspace:*", + "@workflow/core": "workspace:*", + "@workflow/errors": "workspace:*", + "@workflow/utils": "workspace:*", + "@workflow/world": "workspace:*" + }, + "devDependencies": { + "@types/node": "catalog:", + "@workflow/tsconfig": "workspace:*", + "typescript": "catalog:", + "vitest": "catalog:" + } +} diff --git a/packages/world-sim/src/build.ts b/packages/world-sim/src/build.ts new file mode 100644 index 0000000000..5830b2daca --- /dev/null +++ b/packages/world-sim/src/build.ts @@ -0,0 +1,105 @@ +/** + * Bundle a project's workflows for the simulator. + * + * The workflow orchestrator runs from a *code string* inside a VM + * (`workflowEntrypoint(workflowCode)`), so there is no way to hand the runtime + * a live function reference β€” a scenario needs the same compiled combined + * bundle a real deployment would serve. This mirrors what `@workflow/vitest` + * does in its global setup, with one addition: the build's manifest is + * returned, which is how a scenario can name a workflow by its plain function + * name instead of importing a client-transformed reference. + * + * This module reaches SWC and esbuild through `@workflow/builders`, so it is + * deliberately *not* part of the package's main entry β€” see `load.ts`. Import + * it as `@workflow/world-sim/build`, and only from something that compiles. + */ + +import { mkdir } from 'node:fs/promises'; +import { join } from 'node:path'; +import { + BaseBuilder, + createBaseBuilderConfig, + type WorkflowManifest, +} from '@workflow/builders'; + +export interface SimBuildOptions { + /** Project directory containing the workflows to compile. */ + cwd: string; + /** Directories (relative to `cwd`) to scan. Defaults to the project root. */ + dirs?: string[]; + /** Where to write the generated bundles. Defaults to `/.workflow-sim`. */ + outDir?: string; +} + +export interface SimBundle { + /** Absolute path of the combined flow+step bundle. */ + flowBundlePath: string; + manifest: WorkflowManifest; + /** + * Workflow function name β†’ machine workflow id, flattened from the manifest. + * Ambiguous short names (same function name in two files) are omitted in + * favour of their `#` keys, which are always present. + */ + workflowIds: Record; +} + +class SimBuilder extends BaseBuilder { + #outDir: string; + manifest: WorkflowManifest = {}; + + constructor(workingDir: string, outDir: string, dirs: string[]) { + super({ + ...createBaseBuilderConfig({ workingDir, dirs }), + // 'next' emits ESM with Node-compatible output, which is what a plain + // `import()` in this process can load. + buildTarget: 'next', + suppressCreateWorkflowsBundleLogs: true, + suppressCreateWebhookBundleLogs: true, + suppressCreateManifestLogs: true, + }); + this.#outDir = outDir; + } + + override async build(): Promise { + const inputFiles = await this.getInputFiles(); + await mkdir(this.#outDir, { recursive: true }); + const { manifest } = await this.createCombinedBundle({ + inputFiles, + stepsOutfile: join(this.#outDir, '__step_registrations.mjs'), + flowOutfile: join(this.#outDir, 'combined.mjs'), + format: 'esm', + bundleFinalOutput: false, + externalizeNonSteps: true, + // Nothing downstream bundles this output β€” Node imports it directly β€” so + // project-local imports have to be inlined rather than left as bare `.ts` + // specifiers. + bundleTransitiveLocalStepDependencies: true, + }); + this.manifest = manifest; + } +} + +export async function buildSimBundle( + options: SimBuildOptions +): Promise { + const outDir = options.outDir ?? join(options.cwd, '.workflow-sim'); + const builder = new SimBuilder(options.cwd, outDir, options.dirs ?? ['.']); + await builder.build(); + + const workflowIds: Record = {}; + const ambiguous = new Set(); + for (const [file, fns] of Object.entries(builder.manifest.workflows ?? {})) { + for (const [fn, { workflowId }] of Object.entries(fns)) { + workflowIds[`${file}#${fn}`] = workflowId; + if (fn in workflowIds) ambiguous.add(fn); + else workflowIds[fn] = workflowId; + } + } + for (const name of ambiguous) delete workflowIds[name]; + + return { + flowBundlePath: join(outDir, 'combined.mjs'), + manifest: builder.manifest, + workflowIds, + }; +} diff --git a/packages/world-sim/src/clock.test.ts b/packages/world-sim/src/clock.test.ts new file mode 100644 index 0000000000..ac586df273 --- /dev/null +++ b/packages/world-sim/src/clock.test.ts @@ -0,0 +1,83 @@ +import { describe, expect, it } from 'vitest'; +import { createVirtualClock, DEFAULT_EPOCH_MS } from './clock.js'; + +describe('virtual clock', () => { + it('starts at the epoch and only moves when told to', () => { + const clock = createVirtualClock(); + expect(clock.now()).toBe(DEFAULT_EPOCH_MS); + expect(clock.elapsed()).toBe(0); + clock.advanceBy(1500); + expect(clock.elapsed()).toBe(1500); + }); + + it('never moves backwards', () => { + const clock = createVirtualClock(1000); + clock.advanceTo(5000); + clock.advanceTo(2000); + expect(clock.now()).toBe(5000); + }); + + it('rejects negative advances rather than silently rewinding', () => { + const clock = createVirtualClock(); + expect(() => clock.advanceBy(-1)).toThrow(/non-negative/); + }); + + it('drives Date.now() and new Date() while installed, and restores after', () => { + const realNow = Date.now(); + const clock = createVirtualClock(0); + const uninstall = clock.install(); + try { + expect(Date.now()).toBe(0); + // biome-ignore lint/complexity/useDateNow: the point is to check the + // zero-argument constructor, not just the static. + expect(new Date().getTime()).toBe(0); + // Explicit arguments must still behave normally β€” only the "what time + // is it" reading is virtual. + expect(new Date(12345).getTime()).toBe(12345); + expect(new Date() instanceof Date).toBe(true); + clock.advanceBy(60_000); + expect(Date.now()).toBe(60_000); + } finally { + uninstall(); + } + expect(Date.now()).toBeGreaterThanOrEqual(realNow); + }); + + it('keeps `instanceof Date` true across successive clocks', () => { + // The replay check installs its own clock after the scenario's. If each + // install swapped in a fresh Date *subclass*, every Date made under the + // previous one would stop being `instanceof Date` β€” which silently turns + // dates into `{}` in any structural clone. + const first = createVirtualClock(1000); + const uninstallFirst = first.install(); + const madeUnderFirst = new Date(); + uninstallFirst(); + + const second = createVirtualClock(2000); + const uninstallSecond = second.install(); + try { + expect(madeUnderFirst instanceof Date).toBe(true); + expect(madeUnderFirst.getTime()).toBe(1000); + expect(new Date() instanceof Date).toBe(true); + expect(Object.prototype.toString.call(madeUnderFirst)).toBe( + '[object Date]' + ); + } finally { + uninstallSecond(); + } + expect(madeUnderFirst instanceof Date).toBe(true); + }); + + it('refuses to nest, so a leaked scenario cannot shadow the next one', () => { + const first = createVirtualClock(); + const second = createVirtualClock(); + const uninstall = first.install(); + try { + expect(() => second.install()).toThrow(/already installed/); + } finally { + uninstall(); + } + // Once the first is uninstalled the second is free to take over. + second.install()(); + }); +}); diff --git a/packages/world-sim/src/clock.ts b/packages/world-sim/src/clock.ts new file mode 100644 index 0000000000..0eefc617ef --- /dev/null +++ b/packages/world-sim/src/clock.ts @@ -0,0 +1,117 @@ +/** + * Virtual clock. + * + * The runtime decides when a `sleep()` has elapsed by comparing the wait's + * stored `resumeAt` against host `Date.now()` (see the "complete elapsed + * waits" pass in `@workflow/core`'s runtime). A simulation that wants + * scenarios to terminate in milliseconds therefore cannot leave the host + * clock alone: it has to be able to jump the process forward to the next + * scheduled deadline. + * + * So the clock here is authoritative for the whole scenario, and `install()` + * patches `Date.now()` and the zero-argument `Date` constructor to read it. + * Timers are deliberately NOT patched: `@workflow/core` uses + * `setTimeout(fn, 0)` as a macrotask barrier in several ordering-sensitive + * places (`events-consumer.ts`, `private.ts`), and swapping those for fake + * timers would change the very interleavings the simulation exists to + * observe. Real zero-delay timers stay real; only the *readings* of wall + * time move under our control. + * + * The clock never moves on its own β€” only `advanceTo`/`advanceBy` move it, + * and only the scheduler calls those. Two runs of the same scenario see the + * exact same sequence of timestamps. + */ + +/** Default epoch: 2024-01-01T00:00:00.000Z. Arbitrary, but fixed and readable. */ +export const DEFAULT_EPOCH_MS = 1_704_067_200_000; + +export interface VirtualClock { + /** Current virtual time, in epoch milliseconds. */ + now(): number; + /** Milliseconds elapsed since the clock's epoch. */ + elapsed(): number; + /** Move time forward to `ms`. Never moves backwards. */ + advanceTo(ms: number): void; + /** Move time forward by `ms` (must be >= 0). */ + advanceBy(ms: number): void; + /** + * Patch host `Date.now()` / `new Date()` to read this clock. Returns the + * uninstall function. Nested installs are rejected so a leaked scenario + * can't silently shadow the next one's clock. + */ + install(): () => void; +} + +let installedClock: VirtualClock | undefined; + +export function createVirtualClock(epochMs = DEFAULT_EPOCH_MS): VirtualClock { + let current = epochMs; + + const clock: VirtualClock = { + now: () => current, + elapsed: () => current - epochMs, + advanceTo(ms) { + if (ms > current) current = ms; + }, + advanceBy(ms) { + // Integer, not merely finite: the clock is what stamps ULID timestamps, + // and a fractional millisecond does not survive base32 encoding intact. + if (!Number.isInteger(ms) || ms < 0) { + throw new Error( + `advanceBy expects a non-negative whole number of milliseconds, got ${ms}` + ); + } + current += ms; + }, + install() { + if (installedClock && installedClock !== clock) { + throw new Error( + 'A virtual clock is already installed. Scenarios must run one at a time.' + ); + } + installedClock = clock; + + const RealDate = globalThis.Date; + + // A Proxy, deliberately not a subclass. + // + // Subclassing works right up until two clocks are installed in + // succession β€” a scenario's, then the replay check's. Every `Date` the + // first clock produced is an instance of *that* subclass, so once the + // second one is installed `x instanceof Date` is false for all of them, + // and any code branching on it (a structural clone, a serializer) + // silently mistakes a Date for a plain object. A Proxy keeps + // `Date.prototype` identical to the real one, so `instanceof` stays true + // for every Date ever made, whichever clock is installed. + const proxy = new Proxy(RealDate, { + construct(target, args, newTarget) { + // Only the zero-argument "what time is it" form is virtual. + return Reflect.construct( + target, + args.length === 0 ? [current] : args, + newTarget + ); + }, + apply() { + // `Date()` called as a function ignores its arguments and returns + // the current time as a string. + return new RealDate(current).toString(); + }, + get(target, prop, receiver) { + if (prop === 'now') return () => current; + return Reflect.get(target, prop, receiver); + }, + }); + + globalThis.Date = proxy as DateConstructor; + + return () => { + if (installedClock !== clock) return; + globalThis.Date = RealDate; + installedClock = undefined; + }; + }, + }; + + return clock; +} diff --git a/packages/world-sim/src/drive.ts b/packages/world-sim/src/drive.ts new file mode 100644 index 0000000000..7313751663 --- /dev/null +++ b/packages/world-sim/src/drive.ts @@ -0,0 +1,231 @@ +/** + * The scheduler. + * + * Take the next queue message, jump the virtual clock to its delivery time, + * hand it to the flow handler, wait for it to finish, repeat until the queue + * is empty or a budget says stop. That single loop is the whole of "nothing in + * this world happens on its own". + * + * It lives apart from `scenario.ts` because two things drive it: a scenario, + * and the replay verification that cold-starts a second world from the first + * one's committed log. + */ + +import { setTimeout as sleep } from 'node:timers/promises'; +import { createWorkflowUrl } from '@workflow/utils'; +import { encodeMessage, type QueuedMessage } from './queue.js'; +import type { PendingMessageView } from './types.js'; +import type { SimWorld } from './world.js'; + +export interface ScenarioLimits { + /** Maximum queue deliveries before the scenario is abandoned. */ + maxDeliveries?: number; + /** Maximum span of virtual time the scenario may cover. */ + maxVirtualMs?: number; + /** Wall-clock guard against a genuinely non-terminating step body. */ + maxWallMs?: number; + /** + * Wall-clock budget for a single `Writer.runTo*` wait. + * + * Deliberately far below `maxWallMs`: a `runTo` that blows its budget can say + * which writer failed to reach which point and where every other writer was + * standing, and that diagnosis is worth much more than the generic + * "the scenario ran out of wall clock" the global deadline can offer. It is + * clamped to `maxWallMs` so a scenario that lowers the global budget does not + * have to remember to lower this one too. + */ + maxRunToWallMs?: number; +} + +export const DEFAULT_LIMITS: Required = { + maxDeliveries: 200, + // A year of virtual time. Long enough for any realistic sleep chain, short + // enough that a runaway `while (true) { await sleep('1d') }` is caught. + maxVirtualMs: 365 * 24 * 60 * 60 * 1000, + maxWallMs: 60_000, + // Everything in this world is in-memory, so a point that is reachable at all + // is reached in milliseconds. Seconds of grace is generous. + maxRunToWallMs: 5_000, +}; + +/** + * Let the JS event loop settle. + * + * The runtime uses zero-delay macrotasks as ordering barriers in the replay + * consumer, and `waitUntil`-style background work is not awaited by anyone. + * Between deliveries we drain both so the next delivery starts from a quiet + * process β€” otherwise a message enqueued from a trailing microtask would be + * missed and the scenario would report a spurious stall. + */ +async function settle(rounds = 4): Promise { + for (let i = 0; i < rounds; i++) { + await sleep(0); + } +} + +/** Scenario-supplied override for which pending message goes next. */ +export type SelectNext = (pending: PendingMessageView[]) => string | undefined; + +export interface DriveResult { + deliveries: number; + /** Set to the reason when a budget stopped the loop rather than quiescence. */ + exceeded?: string; +} + +/** + * The scheduler. + * + * Take the next message, jump the clock to its delivery time, hand it to the + * flow handler, repeat until the queue is empty or a budget says stop. This is + * the whole of "nothing happens on its own" β€” extracted so the replay + * verification can drive a second world through exactly the same loop. + */ +export async function driveQueue(options: { + world: SimWorld; + limits: Required; + wallStart: number; + selectNext?: SelectNext; +}): Promise { + const { world, limits, wallStart } = options; + let deliveries = 0; + + while (true) { + await settle(); + + if (performanceNow() - wallStart > limits.maxWallMs) { + return { + deliveries, + exceeded: `wall-clock budget exceeded (${limits.maxWallMs}ms) β€” a step body is probably not terminating`, + }; + } + + const message = selectMessage(world, options.selectNext); + if (!message) break; + + if (deliveries >= limits.maxDeliveries) { + world.simQueue.requeue(message, message.readyAtMs); + return { + deliveries, + exceeded: `delivery budget exceeded (${limits.maxDeliveries} deliveries)`, + }; + } + + world.clock.advanceTo(message.readyAtMs); + if (world.clock.elapsed() > limits.maxVirtualMs) { + world.simQueue.requeue(message, message.readyAtMs); + return { + deliveries, + exceeded: `virtual-time budget exceeded (${limits.maxVirtualMs}ms) β€” the run keeps rescheduling itself into the future`, + }; + } + + deliveries++; + await deliver(world, message); + } + + await settle(); + return { deliveries }; +} + +/** + * Pick the next message to deliver: the scenario's choice when it made one and + * that message is still pending, otherwise the default (earliest ready, then + * enqueue order). + */ +function selectMessage( + world: SimWorld, + selectNext: SelectNext | undefined +): QueuedMessage | undefined { + if (selectNext) { + const chosen = selectNext(world.simQueue.view()); + if (chosen) { + const message = world.simQueue.takeById(chosen); + if (message) return message; + world.pushTrace({ + kind: 'warn', + message: `selectNext chose ${chosen}, which is not pending; falling back to the default order`, + }); + } + } + return world.simQueue.takeNext(); +} + +/** + * Hand one message to the flow handler and apply the queue's response + * protocol: `{ timeoutSeconds }` reschedules the same message (same + * `messageId`, which the runtime's inline step-ownership lease depends on), + * a non-2xx redelivers after a backoff, anything else settles it. + */ +export async function deliver( + world: SimWorld, + message: QueuedMessage +): Promise { + const handler = world.simQueue.handlerFor(message.queueName); + if (!handler) { + world.pushTrace({ + kind: 'warn', + message: `no handler registered for queue ${message.queueName}; message dropped`, + }); + world.simQueue.settle(message); + return; + } + + message.deliveries++; + const payload = message.payload as { stepId?: string }; + world.pushTrace({ + kind: 'delivery', + message: `deliver ${message.messageId} attempt ${message.deliveries}${ + payload.stepId ? ` (inline step ${payload.stepId})` : '' + }`, + }); + + const request = new Request( + createWorkflowUrl('http://sim.local', { type: 'flow' }), + { + method: 'POST', + headers: { + 'content-type': 'application/json', + 'x-vqs-queue-name': message.queueName, + 'x-vqs-message-id': message.messageId, + 'x-vqs-message-attempt': String(message.deliveries), + }, + body: encodeMessage(message.payload), + } + ); + + const response = await handler(request); + const text = await response.text(); + + if (response.ok) { + let timeoutSeconds: number | undefined; + try { + const parsed = Number(JSON.parse(text).timeoutSeconds); + if (Number.isFinite(parsed) && parsed >= 0) timeoutSeconds = parsed; + } catch { + // Not a timeout response. + } + if (timeoutSeconds !== undefined) { + world.simQueue.requeue( + message, + world.clock.now() + timeoutSeconds * 1000 + ); + return; + } + world.simQueue.settle(message); + return; + } + + world.pushTrace({ + kind: 'warn', + message: `handler returned HTTP ${response.status} for ${message.messageId}: ${text}`, + }); + // Mirror world-local's flat 5s retry spacing. The real cap lives in the + // runtime (MAX_QUEUE_DELIVERIES); the scenario's delivery budget is the + // backstop. + world.simQueue.requeue(message, world.clock.now() + 5_000); +} + +/** Real elapsed time, immune to the virtual `Date` patch. */ +export function performanceNow(): number { + return performance.now(); +} diff --git a/packages/world-sim/src/ids.test.ts b/packages/world-sim/src/ids.test.ts new file mode 100644 index 0000000000..1fcd36a9a8 --- /dev/null +++ b/packages/world-sim/src/ids.test.ts @@ -0,0 +1,37 @@ +import { ulidToDate, workflowRunIdSchema } from '@workflow/world'; +import { describe, expect, it } from 'vitest'; +import { createIdFactory } from './ids.js'; + +describe('deterministic id factory', () => { + it('mints run ids the rest of the SDK accepts as ULIDs', () => { + let now = 1_704_067_200_000; + const ids = createIdFactory(() => now); + const runId = ids.runId(); + expect(workflowRunIdSchema.safeParse(runId).success).toBe(true); + // `runIdCreatedAt` decodes this to seed the workflow VM's fixed clock, so + // the embedded timestamp has to be the virtual time, not the host's. + expect(ulidToDate(runId.slice('wrun_'.length))?.getTime()).toBe(now); + + now += 5_000; + expect(ulidToDate(ids.runId().slice('wrun_'.length))?.getTime()).toBe(now); + }); + + it('sorts by (time, mint order), which is what event-log ordering relies on', () => { + let now = 1_704_067_200_000; + const ids = createIdFactory(() => now); + const a = ids.eventId(); + const b = ids.eventId(); + now += 1; + const c = ids.eventId(); + expect(a < b).toBe(true); + expect(b < c).toBe(true); + }); + + it('is a pure function of (clock, counter)', () => { + const build = () => { + const ids = createIdFactory(() => 1_704_067_200_000); + return [ids.runId(), ids.eventId(), ids.messageId()]; + }; + expect(build()).toEqual(build()); + }); +}); diff --git a/packages/world-sim/src/ids.ts b/packages/world-sim/src/ids.ts new file mode 100644 index 0000000000..f7a62b227f --- /dev/null +++ b/packages/world-sim/src/ids.ts @@ -0,0 +1,100 @@ +/** + * Deterministic identifier minting. + * + * Every ID the simulation hands out is a function of (virtual time, a + * per-scenario counter) β€” never of `Math.random()` or the host clock. Two + * runs of the same scenario produce byte-identical run IDs, event IDs and + * message IDs, which is what makes an event-stream dump usable as a golden + * file. + * + * IDs still have to be *real* ULIDs: `@workflow/world` validates run IDs with + * `z.string().ulid()` and decodes their embedded timestamp (both to reject + * clock-skewed clients and to seed the workflow VM's fixed clock), so the + * encoding below is the standard Crockford base32 layout β€” 10 timestamp + * characters followed by 16 characters of "randomness" that we fill from the + * counter instead. + */ + +const CROCKFORD = '0123456789ABCDEFGHJKMNPQRSTVWXYZ'; + +function encodeBase32(value: number, length: number): string { + let out = ''; + let remaining = value; + for (let i = length - 1; i >= 0; i--) { + const mod = remaining % 32; + out = CROCKFORD[mod] + out; + remaining = (remaining - mod) / 32; + } + return out; +} + +/** + * Decode the mint time back out of an id, prefixed (`evnt_01H…`) or bare. + * + * Both concurrency guards compare *ULID times*, never row timestamps: the + * client's snapshot is the ULID time of the newest event it loaded, and the + * server's marker is the ULID time of the newest out-of-band event. Anything + * that has to reason about the log's order therefore reads it back out of the + * id, which is why this lives beside the minting and not beside a caller. + * + * Returns `+Infinity` for an id whose time field is not base32 β€” an id that + * cannot be placed sorts after everything rather than silently landing at 0. + */ +export function ulidTimeOf(id: string): number { + const ulid = id.includes('_') ? id.slice(id.indexOf('_') + 1) : id; + let time = 0; + for (const char of ulid.slice(0, 10)) { + const digit = CROCKFORD.indexOf(char); + if (digit === -1) return Number.POSITIVE_INFINITY; + time = time * 32 + digit; + } + return time; +} + +/** + * A monotonic ULID source. + * + * Monotonicity matters beyond aesthetics: `events.list` sorts by + * `(createdAt, eventId)`, and the runtime replays events in that order. Since + * virtual time only moves when the scheduler jumps it, many events share a + * millisecond, and the counter-derived suffix is what keeps their order + * stable and equal to insertion order. + */ +export interface IdFactory { + /** Mint a bare ULID stamped with the current virtual time. */ + ulid(): string; + /** Mint `evnt_`. */ + eventId(): string; + /** Mint `wrun_`. */ + runId(): string; + /** Mint a monotonically increasing message id. */ + messageId(): string; + /** Number of IDs minted so far β€” also the tiebreak counter. */ + count(): number; +} + +export function createIdFactory(now: () => number): IdFactory { + let counter = 0; + + const ulid = (): string => { + counter++; + // 48-bit timestamp, 10 base32 chars β€” the standard ULID time component. + // `Math.floor` rather than trusting the caller: the clock guards its own + // arithmetic, but `now` is an arbitrary function and a fractional + // millisecond here would silently mint an id that sorts nowhere sensible. + const time = encodeBase32(Math.floor(now()), 10); + // The 16-char entropy component is split so the whole ULID sorts by + // (time, counter): a zero-padded 10-char counter, then a fixed marker + // that makes simulated IDs visually obvious in a dump. + const seq = encodeBase32(counter, 10); + return `${time}${seq}05JM0S`; + }; + + return { + ulid, + eventId: () => `evnt_${ulid()}`, + runId: () => `wrun_${ulid()}`, + messageId: () => `msg_${ulid()}`, + count: () => counter, + }; +} diff --git a/packages/world-sim/src/index.ts b/packages/world-sim/src/index.ts new file mode 100644 index 0000000000..3724d07db0 --- /dev/null +++ b/packages/world-sim/src/index.ts @@ -0,0 +1,63 @@ +/** + * `@workflow/world-sim` β€” a deterministic, fully in-memory World for playing + * out workflow scenarios and checking the world contract holds. + * + * See the package README for the model. The short version: nothing in this + * world happens on its own, every World method is a point a scenario can + * inject at, and virtual time means a scenario that sleeps for a month still + * finishes in milliseconds. + * + * This entry is the *scenario* surface: write one, play it, render the result, + * and name anything those three hand you. The pieces that build or inspect the + * simulator itself β€” `createSimWorld`, `createSimStore`, `driveQueue`, + * `verifyReplay`, `checkInvariants`, the clock β€” are deliberately not here. + * Nothing outside the package has wanted them, and re-exporting them makes + * every one of their signatures a compatibility promise. Import them from their + * module if you are extending the simulator; see `DESIGN.md`. + */ + +// `buildSimBundle` is deliberately absent too, for a different reason: it +// reaches SWC and esbuild through `@workflow/builders`, and a consumer that +// only wants to *play* scenarios should not drag a compiler into its module +// graph. It is exported from `@workflow/world-sim/build` instead. +export type { SelectNext } from './drive.js'; +export { loadFlowHandler } from './load.js'; +export { + type MarkdownSummaryOptions, + type RenderOptions, + renderMarkdownSummary, + renderScenario, + renderSummary, +} from './report.js'; +export { + type RunScenarioOptions, + runScenario, + type ScenarioExpectation, + type ScenarioOutcome, + type ScenarioResult, + type ScenarioSpec, +} from './scenario.js'; +export { ScenarioAborted } from './tempo.js'; +export type { + CallContext, + CallMatch, + CallPhase, + Held, + InFlightWrite, + InvariantViolation, + ObservedPoint, + Parked, + PendingMessageView, + RejectedCall, + RunToOptions, + ScenarioApi, + ScenarioScript, + Tempo, + TraceEntry, + WorldCallName, + WorldSnapshot, + Writer, + WriterHandles, + WriterId, +} from './types.js'; +export { AlreadyPassedError, RunToTimeoutError } from './writers.js'; diff --git a/packages/world-sim/src/invariants.test.ts b/packages/world-sim/src/invariants.test.ts new file mode 100644 index 0000000000..1e9eb79745 --- /dev/null +++ b/packages/world-sim/src/invariants.test.ts @@ -0,0 +1,491 @@ +import type { Event, Step, WorkflowRun } from '@workflow/world'; +import { describe, expect, it } from 'vitest'; +import { checkInvariants, type InvariantInput } from './invariants.js'; + +/** + * Most cases here do not care about the commit-order/log-order distinction, so + * they pass one array and get it used for both. The cases that do care pass + * `eventsInCommitOrder` explicitly. + */ +function check( + input: Omit & + Partial> +) { + return checkInvariants({ + eventsInCommitOrder: input.events, + ...input, + }); +} + +const RUN = 'wrun_01HK153X00000000000105JM0S'; +const BASE = new Date('2024-01-01T00:00:00.000Z'); + +let counter = 0; +function event(partial: Partial & Pick): Event { + counter++; + return { + runId: RUN, + eventId: `evnt_${String(counter).padStart(4, '0')}`, + createdAt: new Date(BASE.getTime() + counter), + specVersion: 5, + ...partial, + } as Event; +} + +function run(status: WorkflowRun['status']): WorkflowRun { + return { + runId: RUN, + deploymentId: 'dpl_sim', + workflowName: 'workflow//./w//demo', + status, + attributes: {}, + createdAt: BASE, + updatedAt: BASE, + ...(status === 'completed' || status === 'failed' || status === 'cancelled' + ? { completedAt: BASE } + : {}), + ...(status === 'completed' ? { output: new Uint8Array() } : {}), + ...(status === 'failed' ? { error: new Uint8Array() } : {}), + } as WorkflowRun; +} + +const rules = (violations: { rule: string }[]) => violations.map((v) => v.rule); + +describe('invariants', () => { + it('accepts a well-formed run', () => { + const events = [ + event({ + eventType: 'run_created', + eventData: { + deploymentId: 'd', + workflowName: 'w', + input: new Uint8Array(), + }, + }), + event({ eventType: 'run_started' }), + event({ + eventType: 'step_created', + correlationId: 's1', + eventData: { stepName: 'step//./w//a', input: new Uint8Array() }, + }), + event({ eventType: 'step_started', correlationId: 's1' }), + event({ + eventType: 'step_completed', + correlationId: 's1', + eventData: { result: new Uint8Array() }, + }), + event({ + eventType: 'run_completed', + eventData: { output: new Uint8Array() }, + }), + ]; + const steps: Step[] = [ + { + runId: RUN, + stepId: 's1', + stepName: 'step//./w//a', + status: 'completed', + attempt: 1, + createdAt: BASE, + updatedAt: BASE, + }, + ]; + expect( + check({ + runId: RUN, + events, + runs: [run('completed')], + steps, + waits: [], + }) + ).toEqual([]); + }); + + it('catches a log that gained a row behind a committed peer', () => { + // Log order is fine; commit order is not. `b` took a position above `a` + // and then `a` committed into the gap β€” the shape an append-only log + // promises cannot happen. + const a = event({ + eventType: 'run_created', + eventData: { + deploymentId: 'd', + workflowName: 'w', + input: new Uint8Array(), + }, + }); + const b = event({ eventType: 'run_started' }); + expect( + rules( + check({ + runId: RUN, + events: [a, b], + eventsInCommitOrder: [b, a], + runs: [run('running')], + steps: [], + waits: [], + }) + ) + ).toContain('log.monotonic-order'); + }); + + it('skips the order rule when the world makes no such promise', () => { + const a = event({ + eventType: 'run_created', + eventData: { + deploymentId: 'd', + workflowName: 'w', + input: new Uint8Array(), + }, + }); + const b = event({ eventType: 'run_started' }); + expect( + rules( + checkInvariants({ + runId: RUN, + events: [a, b], + runs: [run('running')], + steps: [], + waits: [], + }) + ) + ).not.toContain('log.monotonic-order'); + }); + + it('catches an out-of-order log', () => { + const a = event({ + eventType: 'run_created', + eventData: { + deploymentId: 'd', + workflowName: 'w', + input: new Uint8Array(), + }, + }); + const b = event({ eventType: 'run_started' }); + const violations = check({ + runId: RUN, + // Appended in the wrong order relative to how `events.list` will sort + // them: replay would see a different sequence than what happened. + events: [b, a], + runs: [run('running')], + steps: [], + waits: [], + }); + expect(rules(violations)).toContain('log.monotonic-order'); + expect(rules(violations)).toContain('run.created-first'); + }); + + it('catches a step restarted after it finished', () => { + const events = [ + event({ + eventType: 'run_created', + eventData: { + deploymentId: 'd', + workflowName: 'w', + input: new Uint8Array(), + }, + }), + event({ + eventType: 'step_created', + correlationId: 's1', + eventData: { stepName: 'a', input: new Uint8Array() }, + }), + event({ eventType: 'step_started', correlationId: 's1' }), + event({ + eventType: 'step_completed', + correlationId: 's1', + eventData: { result: new Uint8Array() }, + }), + event({ eventType: 'step_started', correlationId: 's1' }), + ]; + expect( + rules( + check({ + runId: RUN, + events, + runs: [run('running')], + steps: [], + waits: [], + }) + ) + ).toContain('step.no-restart-after-terminal'); + }); + + it('catches an entity that disagrees with the log', () => { + const events = [ + event({ + eventType: 'run_created', + eventData: { + deploymentId: 'd', + workflowName: 'w', + input: new Uint8Array(), + }, + }), + event({ + eventType: 'step_created', + correlationId: 's1', + eventData: { stepName: 'a', input: new Uint8Array() }, + }), + event({ eventType: 'step_started', correlationId: 's1' }), + ]; + const steps: Step[] = [ + { + runId: RUN, + stepId: 's1', + stepName: 'a', + status: 'completed', + attempt: 4, + createdAt: BASE, + updatedAt: BASE, + }, + ]; + const violations = rules( + check({ + runId: RUN, + events, + runs: [run('running')], + steps, + waits: [], + }) + ); + expect(violations).toContain('step.entity-matches-log'); + expect(violations).toContain('step.attempt-matches-log'); + }); + + it('catches a run whose attributes disagree with its attr_set events', () => { + const events = [ + event({ + eventType: 'run_created', + eventData: { + deploymentId: 'd', + workflowName: 'w', + input: new Uint8Array(), + }, + }), + event({ + eventType: 'attr_set', + correlationId: 'a1', + eventData: { + changes: [{ key: 'approval', value: 'yes' }], + writer: { type: 'workflow' }, + }, + }), + ]; + const drifted = { ...run('running'), attributes: { approval: 'no' } }; + expect( + rules( + check({ + runId: RUN, + events, + runs: [drifted], + steps: [], + waits: [], + }) + ) + ).toContain('run.attributes-match-log'); + + const agreeing = { ...run('running'), attributes: { approval: 'yes' } }; + expect( + rules( + check({ + runId: RUN, + events, + runs: [agreeing], + steps: [], + waits: [], + }) + ) + ).not.toContain('run.attributes-match-log'); + }); + + it('honours a removal recorded as a null change', () => { + const events = [ + event({ + eventType: 'run_created', + eventData: { + deploymentId: 'd', + workflowName: 'w', + input: new Uint8Array(), + attributes: { seeded: 'v' }, + }, + }), + event({ + eventType: 'attr_set', + correlationId: 'a1', + eventData: { + changes: [{ key: 'seeded', value: null }], + writer: { type: 'workflow' }, + }, + }), + ]; + expect( + rules( + check({ + runId: RUN, + events, + runs: [{ ...run('running'), attributes: {} }], + steps: [], + waits: [], + }) + ) + ).not.toContain('run.attributes-match-log'); + }); + + it('catches two live hooks holding one token', () => { + const events = [ + event({ + eventType: 'run_created', + eventData: { + deploymentId: 'd', + workflowName: 'w', + input: new Uint8Array(), + }, + }), + event({ + eventType: 'hook_created', + correlationId: 'h1', + eventData: { token: 't' }, + }), + event({ + eventType: 'hook_created', + correlationId: 'h2', + eventData: { token: 't' }, + }), + ]; + expect( + rules( + check({ + runId: RUN, + events, + runs: [run('running')], + steps: [], + waits: [], + }) + ) + ).toContain('hook.token-unique'); + }); + + it('allows a hook token to be reclaimed after disposal', () => { + const events = [ + event({ + eventType: 'run_created', + eventData: { + deploymentId: 'd', + workflowName: 'w', + input: new Uint8Array(), + }, + }), + event({ + eventType: 'hook_created', + correlationId: 'h1', + eventData: { token: 't' }, + }), + event({ eventType: 'hook_disposed', correlationId: 'h1' }), + event({ + eventType: 'hook_created', + correlationId: 'h2', + eventData: { token: 't' }, + }), + ]; + expect( + rules( + check({ + runId: RUN, + events, + runs: [run('running')], + steps: [], + waits: [], + }) + ) + ).not.toContain('hook.token-unique'); + }); + + it('catches a rewritten wait deadline', () => { + const events = [ + event({ + eventType: 'run_created', + eventData: { + deploymentId: 'd', + workflowName: 'w', + input: new Uint8Array(), + }, + }), + event({ + eventType: 'wait_created', + correlationId: 'w1', + eventData: { resumeAt: new Date('2024-02-01T00:00:00Z') }, + }), + event({ + eventType: 'wait_completed', + correlationId: 'w1', + eventData: { resumeAt: new Date('2024-03-01T00:00:00Z') }, + }), + ]; + expect( + rules( + check({ + runId: RUN, + events, + runs: [run('running')], + steps: [], + waits: [], + }) + ) + ).toContain('wait.resume-at-stable'); + }); + + it('allows a running step to close out after the run ended, but nothing else', () => { + const ok = [ + event({ + eventType: 'run_created', + eventData: { + deploymentId: 'd', + workflowName: 'w', + input: new Uint8Array(), + }, + }), + event({ + eventType: 'step_created', + correlationId: 's1', + eventData: { stepName: 'a', input: new Uint8Array() }, + }), + event({ eventType: 'step_started', correlationId: 's1' }), + event({ eventType: 'run_cancelled' }), + event({ + eventType: 'step_completed', + correlationId: 's1', + eventData: { result: new Uint8Array() }, + }), + ]; + expect( + rules( + check({ + runId: RUN, + events: ok, + runs: [run('cancelled')], + steps: [], + waits: [], + }) + ) + ).not.toContain('run.terminal-is-last'); + + const bad = [ + ...ok, + event({ + eventType: 'hook_created', + correlationId: 'h1', + eventData: { token: 't' }, + }), + ]; + expect( + rules( + check({ + runId: RUN, + events: bad, + runs: [run('cancelled')], + steps: [], + waits: [], + }) + ) + ).toContain('run.terminal-is-last'); + }); +}); diff --git a/packages/world-sim/src/invariants.ts b/packages/world-sim/src/invariants.ts new file mode 100644 index 0000000000..f6a9d7b8d8 --- /dev/null +++ b/packages/world-sim/src/invariants.ts @@ -0,0 +1,409 @@ +/** + * Consistency checks over a finished (or stalled) run. + * + * These are the properties the rest of the system assumes without ever + * verifying: that the log is ordered, that entity rows are a pure fold of the + * log, that a step is never restarted after it finished, that a terminal run + * accepts nothing afterwards. The store enforces most of them at write time by + * rejecting bad events β€” but "the store rejected it" and "the log is actually + * consistent" are different claims, and only the second one is worth trusting. + * So this module re-derives everything from the event log alone and compares. + * + * A violation is a bug somewhere: in the runtime that produced the sequence, + * in the store that accepted it, or in the scenario that injected something + * impossible. Which one is a question for the reader; the checker's job is + * only to notice. + */ + +import { + type Event, + isTerminalRunEventType, + type Step, + type Wait, + type WorkflowRun, +} from '@workflow/world'; +import type { InvariantViolation } from './types.js'; + +export interface InvariantInput { + runId: string; + /** The run's events in log order β€” the order every reader sees them in. */ + events: Event[]; + /** + * The same events in the order they were *committed*, supplied only by a + * world that promises the two orders agree β€” i.e. an append-only log. + * + * Only `log.monotonic-order` reads it, and it has to: comparing the sorted + * array against sort order can only ever pass, which is why that rule was + * unfirable before. Under a mint-ordered log the field is omitted and the + * rule is skipped, because there an out-of-order commit is the premise the + * scenario deliberately injected β€” production mints ids at the handler + * boundary, so its log gains rows in the past by design. Asserting otherwise + * would fail every scenario that holds a write across a peer's commit, which + * is the setup, not the fault. + */ + eventsInCommitOrder?: Event[]; + runs: WorkflowRun[]; + steps: Step[]; + waits: Wait[]; +} + +/** Entity state derived purely from the event log. */ +interface DerivedStep { + stepId: string; + stepName?: string; + status: 'pending' | 'running' | 'completed' | 'failed'; + attempt: number; +} + +export function checkInvariants(input: InvariantInput): InvariantViolation[] { + const violations: InvariantViolation[] = []; + const { runId, events } = input; + + const add = (rule: string, message: string, eventId?: string) => { + violations.push({ rule, message, runId, eventId }); + }; + + // ---- Log shape -------------------------------------------------------- + + const seenEventIds = new Set(); + for (const event of events) { + if (seenEventIds.has(event.eventId)) { + add( + 'log.unique-event-id', + `Duplicate eventId ${event.eventId}`, + event.eventId + ); + } + seenEventIds.add(event.eventId); + } + + // `events.list` sorts by (createdAt, eventId), and replay consumes events in + // that order. An append-only log promises commit order *is* that order; if it + // is not, the log gained a row behind a position readers had already passed, + // so a read taken in between saw a sequence the finished log contradicts. + // Walking the sorted array could never notice β€” it is sorted, so it is + // monotonic by construction. This is the check that the promise was kept. + let previousKey = ''; + for (const event of input.eventsInCommitOrder ?? []) { + const key = `${event.createdAt.toISOString()}|${event.eventId}`; + if (previousKey && key <= previousKey) { + add( + 'log.monotonic-order', + `Event ${event.eventId} committed after a peer that sorts above it (${key} <= ${previousKey})`, + event.eventId + ); + } + previousKey = key; + } + + if (events.length > 0 && events[0].eventType !== 'run_created') { + add( + 'run.created-first', + `First event is ${events[0].eventType}, expected run_created`, + events[0].eventId + ); + } + + const createdCount = events.filter( + (e) => e.eventType === 'run_created' + ).length; + if (createdCount > 1) { + add('run.created-once', `${createdCount} run_created events for one run`); + } + + const terminalIndex = events.findIndex((e) => + isTerminalRunEventType(e.eventType) + ); + if (terminalIndex >= 0 && terminalIndex !== events.length - 1) { + const trailing = events.slice(terminalIndex + 1); + // A step that was already running when the run ended is allowed to report + // its terminal write afterwards; everything else is a contract break. + const disallowed = trailing.filter( + (e) => e.eventType !== 'step_completed' && e.eventType !== 'step_failed' + ); + if (disallowed.length > 0) { + add( + 'run.terminal-is-last', + `${disallowed.length} event(s) recorded after the run reached a terminal state: ${disallowed + .map((e) => e.eventType) + .join(', ')}`, + disallowed[0].eventId + ); + } + } + + // ---- Step lifecycle --------------------------------------------------- + + const derivedSteps = new Map(); + for (const event of events) { + const id = event.correlationId; + if (!id) continue; + + switch (event.eventType) { + case 'step_created': { + if (derivedSteps.has(id)) { + add('step.created-once', `Step ${id} created twice`, event.eventId); + } + derivedSteps.set(id, { + stepId: id, + stepName: event.eventData?.stepName, + status: 'pending', + attempt: 0, + }); + break; + } + case 'step_started': { + const step = derivedSteps.get(id); + if (!step) { + add( + 'step.started-after-created', + `step_started for ${id} with no preceding step_created`, + event.eventId + ); + break; + } + if (step.status === 'completed' || step.status === 'failed') { + add( + 'step.no-restart-after-terminal', + `Step ${id} restarted after reaching "${step.status}"`, + event.eventId + ); + } + step.status = 'running'; + step.attempt++; + break; + } + case 'step_completed': + case 'step_failed': { + const step = derivedSteps.get(id); + if (!step) { + add( + 'step.terminal-after-created', + `${event.eventType} for ${id} with no preceding step_created`, + event.eventId + ); + break; + } + if (step.status === 'completed' || step.status === 'failed') { + add( + 'step.terminal-once', + `Step ${id} reached a terminal state twice (${step.status} then ${event.eventType})`, + event.eventId + ); + } + step.status = + event.eventType === 'step_completed' ? 'completed' : 'failed'; + break; + } + case 'step_retrying': { + const step = derivedSteps.get(id); + if (step) step.status = 'pending'; + break; + } + default: + break; + } + } + + // ---- Materialized view agrees with the log ---------------------------- + + for (const step of input.steps) { + const derived = derivedSteps.get(step.stepId); + if (!derived) { + add( + 'step.entity-has-log', + `Step entity ${step.stepId} exists but the log has no step_created for it` + ); + continue; + } + if (derived.status !== step.status) { + add( + 'step.entity-matches-log', + `Step ${step.stepId} entity status "${step.status}" disagrees with the log ("${derived.status}")` + ); + } + if (derived.attempt !== step.attempt) { + add( + 'step.attempt-matches-log', + `Step ${step.stepId} entity attempt ${step.attempt} disagrees with ${derived.attempt} step_started events` + ); + } + } + + // ---- Hooks ------------------------------------------------------------- + + const createdHooks = new Map(); // hookId β†’ token + const disposedHookIds = new Set(); + const liveTokens = new Map(); // token β†’ hookId + + for (const event of events) { + const id = event.correlationId; + if (!id) continue; + + switch (event.eventType) { + case 'hook_created': { + const token = event.eventData.token; + const owner = liveTokens.get(token); + if (owner && owner !== id) { + add( + 'hook.token-unique', + `Token "${token}" was granted to hook ${id} while hook ${owner} still held it`, + event.eventId + ); + } + createdHooks.set(id, token); + liveTokens.set(token, id); + break; + } + case 'hook_received': { + if (!createdHooks.has(id)) { + add( + 'hook.received-after-created', + `hook_received for ${id} with no preceding hook_created`, + event.eventId + ); + } + if (disposedHookIds.has(id)) { + add( + 'hook.no-receive-after-dispose', + `hook_received for ${id} after it was disposed`, + event.eventId + ); + } + break; + } + case 'hook_disposed': { + if (disposedHookIds.has(id)) { + add('hook.dispose-once', `Hook ${id} disposed twice`, event.eventId); + } + disposedHookIds.add(id); + const token = createdHooks.get(id); + if (token && liveTokens.get(token) === id) liveTokens.delete(token); + break; + } + default: + break; + } + } + + // ---- Waits ------------------------------------------------------------- + + const openWaits = new Map(); + const completedWaits = new Set(); + for (const event of events) { + const id = event.correlationId; + if (!id) continue; + if (event.eventType === 'wait_created') { + if (openWaits.has(id) || completedWaits.has(id)) { + add('wait.created-once', `Wait ${id} created twice`, event.eventId); + } + openWaits.set(id, event.eventData.resumeAt); + } else if (event.eventType === 'wait_completed') { + if (!openWaits.has(id)) { + add( + 'wait.completed-after-created', + `wait_completed for ${id} with no preceding wait_created`, + event.eventId + ); + } + if (completedWaits.has(id)) { + add('wait.completed-once', `Wait ${id} completed twice`, event.eventId); + } + const expected = openWaits.get(id); + const actual = event.eventData?.resumeAt; + // The workflow's sleep consumer treats a mismatched `resumeAt` as replay + // divergence, so a world that rewrites it breaks replay rather than + // merely reporting the wrong time. + if ( + expected && + actual && + expected.getTime() !== new Date(actual).getTime() + ) { + add( + 'wait.resume-at-stable', + `wait_completed for ${id} carries resumeAt ${new Date(actual).toISOString()}, but wait_created said ${expected.toISOString()}`, + event.eventId + ); + } + completedWaits.add(id); + openWaits.delete(id); + } + } + + // ---- Attributes are a fold of the log ---------------------------------- + // + // Attributes are the only mutable run state a workflow writes, and the world + // materializes them by applying `attr_set` changes in log order. If the + // materialized map and the log disagree, an observability surface and a + // replay see different run state. + const runEntity = input.runs.find((r) => r.runId === runId); + if (runEntity) { + const created = events.find((e) => e.eventType === 'run_created'); + const derivedAttributes: Record = { + ...(created?.eventData?.attributes ?? {}), + }; + for (const event of events) { + if (event.eventType !== 'attr_set') continue; + for (const change of event.eventData.changes) { + if (change.value === null) delete derivedAttributes[change.key]; + else derivedAttributes[change.key] = change.value; + } + } + const actual = runEntity.attributes ?? {}; + const keys = new Set([ + ...Object.keys(derivedAttributes), + ...Object.keys(actual), + ]); + for (const key of keys) { + if (derivedAttributes[key] !== actual[key]) { + add( + 'run.attributes-match-log', + `Attribute ${JSON.stringify(key)} is ${JSON.stringify(actual[key])} on the run entity but ${JSON.stringify(derivedAttributes[key])} by the log` + ); + } + } + } + + // ---- Run entity agrees with the log ------------------------------------ + + const run = runEntity; + if (run) { + const terminal = events.find((e) => isTerminalRunEventType(e.eventType)); + const expectedStatus = terminal + ? terminal.eventType === 'run_completed' + ? 'completed' + : terminal.eventType === 'run_failed' + ? 'failed' + : 'cancelled' + : events.some((e) => e.eventType === 'run_started') + ? 'running' + : 'pending'; + if (run.status !== expectedStatus) { + add( + 'run.entity-matches-log', + `Run entity status "${run.status}" disagrees with the log ("${expectedStatus}")` + ); + } + if (run.status === 'completed' && run.output === undefined) { + // `run_completed` may legitimately carry no output (a void workflow), so + // this only fires when the event had one and the entity lost it. + const completed = events.find((e) => e.eventType === 'run_completed'); + if (completed?.eventData?.output !== undefined) { + add('run.output-materialized', 'Completed run entity has no output'); + } + } + // Once a run is terminal its hooks and waits are unreachable, so no world + // should still be holding them. + if (terminal) { + const strayWaits = input.waits.filter((w) => w.runId === runId); + if (strayWaits.length > 0) { + add( + 'run.resources-released', + `${strayWaits.length} wait(s) still registered on a terminal run` + ); + } + } + } + + return violations; +} diff --git a/packages/world-sim/src/load.ts b/packages/world-sim/src/load.ts new file mode 100644 index 0000000000..bf1c547080 --- /dev/null +++ b/packages/world-sim/src/load.ts @@ -0,0 +1,47 @@ +/** + * Loading a built bundle, separated from building one. + * + * These two halves have very different dependency footprints. Building pulls + * in `@workflow/builders`, and through it SWC and esbuild β€” tens of megabytes + * of native binaries. Loading needs nothing but `import()`. + * + * Keeping them in one module meant anything that wanted to *run* a bundle also + * dragged the whole compiler into its graph. That is fine in a CLI and + * expensive in a deployed function, so the split is the package's way of + * letting a consumer take the runtime half alone: `@workflow/world-sim` + * re-exports this file, while `buildSimBundle` lives behind + * `@workflow/world-sim/build`. + */ + +import { pathToFileURL } from 'node:url'; + +/** + * Import a built bundle's `POST` handler. + * + * Deliberately eager (unlike `@workflow/vitest`, which defers the import so + * `vi.mock` can still intercept step dependencies): a scenario wants the + * module graph settled before the clock is patched and the first delivery + * runs, so that import-time work never lands in the middle of a measured + * sequence. + * + * The path is only known at runtime β€” it is either a file this process wrote + * seconds ago or one `next build` left on disk β€” so the ignore hints are + * load-bearing wherever a bundler is in the graph. Without them a bundler + * tries to resolve the specifier at build time and fails with "expression is + * too dynamic". They are inert comments under plain Node. + */ +export async function loadFlowHandler( + flowBundlePath: string +): Promise<(req: Request) => Promise> { + const mod = await import( + /* webpackIgnore: true */ /* turbopackIgnore: true */ + pathToFileURL(flowBundlePath).href + ); + const handler = mod.POST; + if (typeof handler !== 'function') { + throw new Error( + `Bundle at ${flowBundlePath} does not export a POST handler. Did the build succeed?` + ); + } + return handler as (req: Request) => Promise; +} diff --git a/packages/world-sim/src/queue.test.ts b/packages/world-sim/src/queue.test.ts new file mode 100644 index 0000000000..c595922cbc --- /dev/null +++ b/packages/world-sim/src/queue.test.ts @@ -0,0 +1,99 @@ +import { ValidQueueName } from '@workflow/world'; +import { describe, expect, it } from 'vitest'; +import { createIdFactory } from './ids.js'; +import { createSimQueue } from './queue.js'; + +const TOPIC = ValidQueueName.parse('__wkf_workflow_workflow//./w//demo'); + +function setup() { + let now = 1_704_067_200_000; + const queue = createSimQueue({ + now: () => now, + ids: createIdFactory(() => now), + deploymentId: 'dpl_sim', + }); + return { queue, advance: (ms: number) => (now += ms), nowMs: () => now }; +} + +describe('sim queue', () => { + it('records messages without delivering anything', async () => { + const { queue } = setup(); + await queue.queue(TOPIC, { runId: 'wrun_a' }); + await queue.queue(TOPIC, { runId: 'wrun_b' }); + expect(queue.pending()).toHaveLength(2); + }); + + it('delivers in (readyAt, enqueue order), so ties are still total', async () => { + const { queue } = setup(); + await queue.queue(TOPIC, { runId: 'later' }, { delaySeconds: 60 }); + await queue.queue(TOPIC, { runId: 'first' }); + await queue.queue(TOPIC, { runId: 'second' }); + + expect(queue.takeNext()?.payload).toMatchObject({ runId: 'first' }); + expect(queue.takeNext()?.payload).toMatchObject({ runId: 'second' }); + expect(queue.takeNext()?.payload).toMatchObject({ runId: 'later' }); + expect(queue.takeNext()).toBeUndefined(); + }); + + it('turns delaySeconds into a virtual ready time', async () => { + const { queue, nowMs } = setup(); + await queue.queue(TOPIC, { runId: 'wrun_a' }, { delaySeconds: 90 }); + expect(queue.pending()[0].readyAtMs).toBe(nowMs() + 90_000); + }); + + it('dedupes on an idempotency key until the message settles', async () => { + const { queue } = setup(); + const a = await queue.queue( + TOPIC, + { runId: 'wrun_a' }, + { idempotencyKey: 'k' } + ); + const b = await queue.queue( + TOPIC, + { runId: 'wrun_a' }, + { idempotencyKey: 'k' } + ); + expect(b.messageId).toBe(a.messageId); + expect(queue.pending()).toHaveLength(1); + + // Wait-continuation logic depends on the key being reusable once the + // message is done β€” a wider dedupe window silently drops re-enqueues. + const message = queue.takeNext(); + if (!message) throw new Error('expected a pending message'); + queue.settle(message); + const c = await queue.queue( + TOPIC, + { runId: 'wrun_a' }, + { idempotencyKey: 'k' } + ); + expect(c.messageId).not.toBe(a.messageId); + }); + + it('keeps the messageId stable across redeliveries', async () => { + const { queue, nowMs } = setup(); + await queue.queue(TOPIC, { runId: 'wrun_a' }); + const first = queue.takeNext(); + if (!first) throw new Error('expected a pending message'); + // Inline step ownership uses the messageId as a liveness lease, so a + // redelivery that minted a fresh id would break crash recovery. + queue.requeue(first, nowMs() + 5_000); + expect(queue.pending()[0].messageId).toBe(first.messageId); + }); + + it('round-trips Uint8Array payloads through the wire encoding', async () => { + const { queue } = setup(); + await queue.queue(TOPIC, { + runId: 'wrun_a', + runInput: { + input: new Uint8Array([1, 2, 3]), + deploymentId: 'dpl_sim', + workflowName: 'workflow//./w//demo', + specVersion: 5, + }, + }); + const message = queue.takeNext(); + const payload = message?.payload as { runInput: { input: Uint8Array } }; + expect(payload.runInput.input).toBeInstanceOf(Uint8Array); + expect([...payload.runInput.input]).toEqual([1, 2, 3]); + }); +}); diff --git a/packages/world-sim/src/queue.ts b/packages/world-sim/src/queue.ts new file mode 100644 index 0000000000..06b75b6996 --- /dev/null +++ b/packages/world-sim/src/queue.ts @@ -0,0 +1,228 @@ +/** + * Deterministic queue. + * + * `@workflow/world-local`'s queue fires a detached async delivery loop from + * inside `queue()`, so the moment a message is enqueued it is racing whatever + * the caller does next. That is faithful to production and useless for a + * simulation: the interleaving is picked by the event loop, not by the test. + * + * Here `queue()` only *records* a message. Nothing is ever delivered until the + * scheduler asks for the next one, and the scheduler always takes the same one: + * the minimum by `(readyAtMs, enqueueSeq)`. Delays are virtual β€” a message + * scheduled 23 hours out is delivered by jumping the clock, not by waiting β€” + * which is what lets a scenario containing `sleep('30d')` finish in + * microseconds. + */ + +import { + MessageId, + parseQueueName, + type Queue, + type QueueOptions, + type QueuePayload, + type QueuePrefix, + ValidQueueName, +} from '@workflow/world'; +import type { IdFactory } from './ids.js'; +import type { PendingMessageView } from './types.js'; + +export interface QueuedMessage { + messageId: string; + queueName: ValidQueueName; + payload: QueuePayload; + readyAtMs: number; + /** Enqueue order; the tiebreak that makes delivery order total. */ + seq: number; + /** Delivery attempts already handed to a handler (the `attempt` header is this + 1). */ + deliveries: number; + idempotencyKey?: string; +} + +export type DirectHandler = (req: Request) => Promise; + +export interface SimQueue extends Queue { + registerHandler(prefix: QueuePrefix, handler: DirectHandler): void; + handlerFor(queueName: string): DirectHandler | undefined; + /** Pending messages in delivery order. */ + pending(): QueuedMessage[]; + /** Remove and return the next message to deliver, or undefined when idle. */ + takeNext(): QueuedMessage | undefined; + /** Take a specific pending message, for scenario-chosen delivery order. */ + takeById(messageId: string): QueuedMessage | undefined; + /** Put a message back for a later delivery attempt, preserving its messageId. */ + requeue(message: QueuedMessage, readyAtMs: number): void; + /** Mark a message finished so its idempotency key can be reused. */ + settle(message: QueuedMessage): void; + view(): PendingMessageView[]; +} + +function replacer(_key: string, value: unknown): unknown { + if (value instanceof Uint8Array) { + return { + __type: 'Uint8Array', + data: Buffer.from(value).toString('base64'), + }; + } + return value; +} + +function reviver(_key: string, value: unknown): unknown { + if ( + value !== null && + typeof value === 'object' && + (value as { __type?: string }).__type === 'Uint8Array' && + typeof (value as { data?: unknown }).data === 'string' + ) { + return new Uint8Array( + Buffer.from((value as { data: string }).data, 'base64') + ); + } + return value; +} + +export function encodeMessage(payload: QueuePayload): string { + return JSON.stringify(payload, replacer); +} + +export function decodeMessage(body: string): unknown { + return JSON.parse(body, reviver); +} + +export function createSimQueue(opts: { + now(): number; + ids: IdFactory; + deploymentId: string; +}): SimQueue { + const messages: QueuedMessage[] = []; + const handlers = new Map(); + /** + * Idempotency keys of messages that are enqueued but not yet settled. This + * matches world-local's in-flight-only dedupe window (VQS holds keys for + * longer); the wait-continuation logic in core is written against exactly + * this behaviour, and widening the window here would silently drop the + * re-enqueues it relies on. + */ + const inflightKeys = new Map(); + let seq = 0; + + const queue: Queue['queue'] = async ( + queueName: ValidQueueName, + message: QueuePayload, + options?: QueueOptions + ) => { + if (options?.idempotencyKey) { + const existing = inflightKeys.get(options.idempotencyKey); + if (existing) return { messageId: MessageId.parse(existing) }; + } + + // Round-trip through the wire encoding at enqueue time so a scenario can + // never accidentally hand the handler a live object reference that + // production would have serialized. + const payload = decodeMessage(encodeMessage(message)) as QueuePayload; + + const messageId = opts.ids.messageId(); + const delayMs = Math.max(0, (options?.delaySeconds ?? 0) * 1000); + const entry: QueuedMessage = { + messageId, + queueName, + payload, + readyAtMs: opts.now() + delayMs, + seq: seq++, + deliveries: 0, + idempotencyKey: options?.idempotencyKey, + }; + if (options?.idempotencyKey) { + inflightKeys.set(options.idempotencyKey, messageId); + } + messages.push(entry); + return { messageId: MessageId.parse(messageId) }; + }; + + const createQueueHandler: Queue['createQueueHandler'] = (prefix, handler) => { + return async (req: Request) => { + const queueName = req.headers.get('x-vqs-queue-name'); + const messageId = req.headers.get('x-vqs-message-id'); + const attempt = Number(req.headers.get('x-vqs-message-attempt')); + if (!queueName || !messageId || !Number.isFinite(attempt)) { + return Response.json( + { error: 'Missing required headers' }, + { status: 400 } + ); + } + if (!queueName.startsWith(prefix)) { + return Response.json({ error: 'Unhandled queue' }, { status: 400 }); + } + const body = decodeMessage(await req.text()); + try { + const result = await handler(body, { + attempt, + queueName: ValidQueueName.parse(queueName), + messageId: MessageId.parse(messageId), + }); + if (typeof result?.timeoutSeconds === 'number') { + return Response.json({ timeoutSeconds: result.timeoutSeconds }); + } + return Response.json({ ok: true }); + } catch (error) { + return Response.json(String(error), { status: 500 }); + } + }; + }; + + const orderPending = () => + [...messages].sort((a, b) => + a.readyAtMs !== b.readyAtMs ? a.readyAtMs - b.readyAtMs : a.seq - b.seq + ); + + return { + queue, + createQueueHandler, + async getDeploymentId() { + return opts.deploymentId; + }, + registerHandler(prefix, handler) { + handlers.set(prefix, handler); + }, + handlerFor(queueName) { + const { prefix } = parseQueueName(ValidQueueName.parse(queueName)); + return handlers.get(prefix); + }, + pending: orderPending, + takeNext() { + const ordered = orderPending(); + const next = ordered[0]; + if (!next) return undefined; + messages.splice(messages.indexOf(next), 1); + return next; + }, + takeById(messageId) { + const index = messages.findIndex((m) => m.messageId === messageId); + if (index === -1) return undefined; + return messages.splice(index, 1)[0]; + }, + requeue(message, readyAtMs) { + messages.push({ ...message, readyAtMs, seq: seq++ }); + }, + settle(message) { + if ( + message.idempotencyKey && + inflightKeys.get(message.idempotencyKey) === message.messageId + ) { + inflightKeys.delete(message.idempotencyKey); + } + }, + view() { + return orderPending().map((m) => { + const payload = m.payload as { runId?: string; stepId?: string }; + return { + messageId: m.messageId, + queueName: m.queueName, + runId: payload.runId, + stepId: payload.stepId, + readyAtMs: m.readyAtMs, + deliveries: m.deliveries, + }; + }); + }, + }; +} diff --git a/packages/world-sim/src/replay.test.ts b/packages/world-sim/src/replay.test.ts new file mode 100644 index 0000000000..bc4a56ff73 --- /dev/null +++ b/packages/world-sim/src/replay.test.ts @@ -0,0 +1,169 @@ +/** + * The replay check is only worth having if it fails when it should, so these + * drive `verifyReplay` against handlers that deliberately re-derive the wrong + * thing. The workflow runtime is stubbed out: what is under test here is the + * comparison β€” given a committed log and a cold replay that disagrees with it, + * is the disagreement reported and named correctly? + */ + +import { getWorld } from '@workflow/core/runtime'; +import { + type Event, + SPEC_VERSION_CURRENT, + type WorkflowRun, +} from '@workflow/world'; +import { describe, expect, it } from 'vitest'; +import { DEFAULT_LIMITS } from './drive.js'; +import { verifyReplay } from './replay.js'; + +const RUN = 'wrun_01HK153X00000000000105JM0S'; +const WORKFLOW = 'workflow//./workflows/demo//demoWorkflow'; +const AT = new Date('2024-01-01T00:00:00.000Z'); +const OUTPUT = new Uint8Array([1, 2, 3]); + +let counter = 0; +function event(partial: Partial & Pick): Event { + counter++; + return { + runId: RUN, + eventId: `evnt_${String(counter).padStart(4, '0')}`, + createdAt: new Date(AT.getTime() + counter), + specVersion: SPEC_VERSION_CURRENT, + ...partial, + } as Event; +} + +function committedLog(): Event[] { + return [ + event({ + eventType: 'run_created', + eventData: { + deploymentId: 'dpl_sim', + workflowName: WORKFLOW, + input: new Uint8Array(), + }, + }), + event({ eventType: 'run_started' }), + event({ eventType: 'run_completed', eventData: { output: OUTPUT } }), + ]; +} + +function completedRun(): WorkflowRun { + return { + runId: RUN, + deploymentId: 'dpl_sim', + workflowName: WORKFLOW, + status: 'completed', + output: OUTPUT, + attributes: {}, + createdAt: AT, + updatedAt: AT, + startedAt: AT, + completedAt: AT, + } as WorkflowRun; +} + +/** A stand-in for the runtime: writes whatever terminal event we tell it to. */ +function handlerWriting( + write: ((world: Awaited>) => Promise) | null +) { + return async () => { + if (write) await write(await getWorld()); + return Response.json({ ok: true }); + }; +} + +async function check( + handler: (req: Request) => Promise, + events = committedLog() +) { + return verifyReplay({ + run: completedRun(), + events, + handler, + limits: { ...DEFAULT_LIMITS, maxWallMs: 5_000 }, + }); +} + +const rules = (result: { violations: { rule: string }[] }) => + result.violations.map((v) => v.rule); + +describe('replay verification', () => { + it('passes when the replay re-derives the withheld terminal event', async () => { + const result = await check( + handlerWriting(async (world) => { + await world.events.create(RUN, { + eventType: 'run_completed', + specVersion: SPEC_VERSION_CURRENT, + eventData: { output: OUTPUT }, + }); + }) + ); + expect(result.violations).toEqual([]); + expect(result.regenerated.map((e) => e.eventType)).toEqual([ + 'run_completed', + ]); + }); + + it('catches a replay that derives a different output', async () => { + const result = await check( + handlerWriting(async (world) => { + await world.events.create(RUN, { + eventType: 'run_completed', + specVersion: SPEC_VERSION_CURRENT, + eventData: { output: new Uint8Array([9, 9, 9]) }, + }); + }) + ); + expect(rules(result)).toContain('replay.output-differs'); + }); + + it('catches a replay that reaches a different terminal state', async () => { + const result = await check( + handlerWriting(async (world) => { + await world.events.create(RUN, { + eventType: 'run_failed', + specVersion: SPEC_VERSION_CURRENT, + eventData: { error: new Uint8Array(), errorCode: 'USER_ERROR' }, + }); + }) + ); + expect(rules(result)).toContain('replay.status-differs'); + expect(rules(result)).toContain('replay.log-differs'); + }); + + it('names a corrupted event log specifically', async () => { + // What the runtime does when a replay cannot follow the history it wrote: + // it exhausts its recovery replays and fails the run with this code. + const result = await check( + handlerWriting(async (world) => { + await world.events.create(RUN, { + eventType: 'run_failed', + specVersion: SPEC_VERSION_CURRENT, + eventData: { + error: new Uint8Array(), + errorCode: 'CORRUPTED_EVENT_LOG', + }, + }); + }) + ); + expect(rules(result)).toEqual(['replay.diverged']); + expect(result.violations[0].message).toMatch( + /could not follow the history it wrote/ + ); + }); + + it('catches a replay that runs out of log and suspends', async () => { + // The handler acknowledges the delivery without finishing the run β€” the + // shape of "the log did not contain enough to rebuild the run". + const result = await check(handlerWriting(null)); + expect(rules(result)).toContain('replay.suspended'); + }); + + it('re-derives nothing to compare when the run never terminated', async () => { + const events = committedLog().slice(0, 2); + const result = await check(handlerWriting(null), events); + expect(result.violations).toEqual([]); + expect(result.regenerated).toEqual([]); + }); +}); diff --git a/packages/world-sim/src/replay.ts b/packages/world-sim/src/replay.ts new file mode 100644 index 0000000000..3999ef7306 --- /dev/null +++ b/packages/world-sim/src/replay.ts @@ -0,0 +1,255 @@ +/** + * Replay verification: does the committed log actually regenerate the state? + * + * Everything else in this package checks the log's *shape* β€” ordering, entity + * rows folding back out of it, lifecycle rules. None of that answers the + * question the durability model actually rests on: if a fresh process picked + * up this log tomorrow, would it reconstruct the same run? + * + * The check is a cold start with the answer withheld. Take the finished log, + * drop its terminal `run_*` event, load the rest into an empty world as + * durable history, and deliver one queue message. The real runtime β€” the same + * `workflowEntrypoint` a deployment serves β€” replays the workflow from the log + * and must re-derive the event we removed, with the same output. No step body + * re-executes (every `step_completed` is in the log, so the step consumer + * resolves from it), so anything the replay produces came from the log alone. + * + * Three ways it can fail, and all three are the same bug class: + * + * - The replay cannot follow the log: the runtime raises + * `ReplayDivergenceError`, retries its recovery replays, and then fails the + * run with `CorruptedEventLogError`. Surfaced here as `replay.diverged`. + * - The replay runs out of log before the workflow finishes, and suspends β€” + * the log did not contain enough to rebuild the run. `replay.suspended`. + * - The replay finishes but derives a different answer. `replay.output-differs` + * / `replay.log-differs`. + */ + +import { setWorld } from '@workflow/core/runtime'; +import { RUN_ERROR_CODES } from '@workflow/errors'; +import { + type Event, + isTerminalRunEventType, + type WorkflowRun, +} from '@workflow/world'; +import { createVirtualClock } from './clock.js'; +import { driveQueue, type ScenarioLimits } from './drive.js'; +import type { InvariantViolation } from './types.js'; +import { createSimWorld, WORKFLOW_QUEUE_PREFIX } from './world.js'; + +export interface ReplayCheckInput { + run: WorkflowRun; + /** The full committed log of the finished run. */ + events: readonly Event[]; + handler: (req: Request) => Promise; + limits: Required; + /** + * Replay under the same log rules as the run being checked. It cannot change + * the outcome β€” the replay seeds a finished log and writes only at a clock + * past its tail, so nothing it appends can be overtaken β€” but a replay + * playing by different rules than the run it verifies is a trap worth not + * setting. + */ + appendOnlyLog?: boolean; +} + +export interface ReplayCheckResult { + violations: InvariantViolation[]; + /** The log the replay produced, for reporting. */ + regenerated: Event[]; + /** How many deliveries the cold replay needed. */ + deliveries: number; +} + +/** Events after the terminal one (a step closing out post-cancellation). */ +function splitAtTerminal(events: readonly Event[]): { + history: Event[]; + terminal: Event | undefined; + trailing: Event[]; +} { + const index = events.findIndex((e) => isTerminalRunEventType(e.eventType)); + if (index === -1) + return { history: [...events], terminal: undefined, trailing: [] }; + return { + history: events.slice(0, index), + terminal: events[index], + trailing: events.slice(index + 1), + }; +} + +/** `(eventType, correlationId)` β€” the part of a log that must be reproducible. */ +function shape(events: readonly Event[]): string[] { + return events.map((e) => + e.correlationId ? `${e.eventType}#${e.correlationId}` : e.eventType + ); +} + +function sameBytes(a: unknown, b: unknown): boolean { + if (a instanceof Uint8Array && b instanceof Uint8Array) { + return a.length === b.length && a.every((byte, i) => byte === b[i]); + } + return a === b; +} + +/** + * Cold-replay a finished run's log and compare what comes back. + * + * Only meaningful for a run that reached a terminal state: a stalled or + * cancelled run's log legitimately does not drive the workflow to an end, so + * there is no regenerated answer to compare against. + */ +export async function verifyReplay( + input: ReplayCheckInput +): Promise { + const violations: InvariantViolation[] = []; + const { run, events, handler, limits } = input; + const runId = run.runId; + + const add = (rule: string, message: string, eventId?: string) => + violations.push({ rule, message, runId, eventId }); + + const { history, terminal, trailing } = splitAtTerminal(events); + if (!terminal) { + return { violations, regenerated: [], deliveries: 0 }; + } + + // A step that closed out after the run terminated cannot be re-derived by a + // replay β€” it was driven by an inline body that already ran, not by the log. + // Seed those as history too so the replay sees the same durable state a + // fresh process would. + const seeded = [...history, ...trailing]; + + // The replay must happen at the instant the run ended, not whenever the + // scenario happened to finish draining its queue. Replaying later is not + // wrong β€” the runtime would legitimately complete any wait that has since + // elapsed β€” but it answers a different question than "does this log + // reproduce this run", and the comparison below would flag the difference as + // a divergence when nothing diverged. + // + // The +1ms offset is what keeps ids sortable: `events.list` orders by + // (createdAt, eventId), so an event minted in the same millisecond as the + // seeded tail could sort ahead of it. One millisecond is below the + // resolution of any wait a workflow can express. + const replayClock = createVirtualClock(terminal.createdAt.getTime() + 1); + const uninstallClock = replayClock.install(); + + const replayWorld = createSimWorld({ + clock: replayClock, + appendOnlyLog: input.appendOnlyLog, + }); + replayWorld.store.seedFromLog(seeded); + replayWorld.registerHandler(WORKFLOW_QUEUE_PREFIX, handler); + replayWorld.setScenarioApi(() => { + throw new Error('the replay world runs no script'); + }); + + setWorld(replayWorld); + + let drain: Awaited>; + try { + await replayWorld.queue( + `${WORKFLOW_QUEUE_PREFIX}${run.workflowName}` as never, + { runId } + ); + drain = await driveQueue({ + world: replayWorld, + limits, + wallStart: performance.now(), + }); + } finally { + uninstallClock(); + } + + const regenerated = replayWorld.store.allEvents(runId).slice(seeded.length); + + if (drain.exceeded) { + add( + 'replay.budget', + `cold replay of the committed log did not settle: ${drain.exceeded}` + ); + return { violations, regenerated, deliveries: drain.deliveries }; + } + + const replayedRun = replayWorld.store + .allRuns() + .find((r) => r.runId === runId); + + // The runtime turns a persistent divergence into a failed run carrying + // REPLAY_DIVERGENCE / CORRUPTED_EVENT_LOG. That is the signal this whole + // check exists to catch, so name it explicitly rather than letting it read + // as a generic "the outcome differs". + const errorCode = replayedRun?.errorCode; + if ( + errorCode === RUN_ERROR_CODES.CORRUPTED_EVENT_LOG || + errorCode === RUN_ERROR_CODES.REPLAY_DIVERGENCE + ) { + add( + 'replay.diverged', + `replaying the committed log failed with ${errorCode}: the runtime could not ` + + `follow the history it wrote. ${await describeRunError(replayedRun?.error)}` + ); + return { violations, regenerated, deliveries: drain.deliveries }; + } + + if ( + !replayedRun || + replayedRun.status === 'running' || + replayedRun.status === 'pending' + ) { + add( + 'replay.suspended', + `replaying the committed log left the run "${replayedRun?.status ?? 'missing'}" instead of "${run.status}" β€” the log does not contain enough to rebuild the run` + ); + return { violations, regenerated, deliveries: drain.deliveries }; + } + + if (replayedRun.status !== run.status) { + add( + 'replay.status-differs', + `replay ended "${replayedRun.status}", the original run ended "${run.status}"` + + (replayedRun.status === 'failed' + ? `: ${await describeRunError(replayedRun.error)}` + : '') + ); + } + + if (!sameBytes(replayedRun.output, run.output)) { + add( + 'replay.output-differs', + 'replay produced a different output than the one recorded on the run' + ); + } + + // The tail the replay re-derived should be the tail we withheld. Compared by + // (eventType, correlationId): ids and timestamps are necessarily new. + const expectedTail = shape([terminal]); + const actualTail = shape(regenerated); + if (actualTail.join(',') !== expectedTail.join(',')) { + add( + 'replay.log-differs', + `replay re-derived [${actualTail.join(', ')}] where the original log recorded [${expectedTail.join(', ')}]` + ); + } + + return { violations, regenerated, deliveries: drain.deliveries }; +} + +/** Best-effort human-readable form of a dehydrated run error, for reporting. */ +async function describeRunError(error: unknown): Promise { + if (error === undefined) return '(no error recorded)'; + try { + // Go through the real hydration path, not the observability one: a run + // error is written by `dehydrateRunError` and may be compressed, which the + // synchronous o11y reviver cannot undo. + const { hydrateRunError } = await import('@workflow/core/serialization'); + const value = await hydrateRunError(error, 'wrun_replay', undefined); + if (value instanceof Error) return `${value.name}: ${value.message}`; + if (value && typeof value === 'object') { + const { name, message } = value as { name?: string; message?: string }; + if (message) return `${name ?? 'Error'}: ${message}`; + } + return JSON.stringify(value) ?? String(value); + } catch { + return '(error could not be hydrated)'; + } +} diff --git a/packages/world-sim/src/report.ts b/packages/world-sim/src/report.ts new file mode 100644 index 0000000000..91d33daeb5 --- /dev/null +++ b/packages/world-sim/src/report.ts @@ -0,0 +1,589 @@ +/** + * Rendering a scenario as text. + * + * The event stream is the primary artifact β€” it is the thing a reader checks + * to answer "did the hook really land between `step_started` and the replay + * that followed it?". So cues, deliveries and notes are interleaved into the + * same column as events rather than printed as a separate log: the whole + * point is their position relative to the events around them. + * + * **Events are referred to one way and one way only: by log position.** `#12` + * is the twelfth event in the durable log β€” the log sorted the way + * `events.list` sorts it, `(createdAt, eventId)` β€” and `@7` is a reference to + * the resource created at position 7. Raw ULIDs never appear; the ids in + * violation messages are rewritten to positions on the way out. One scheme, + * so "the hook is at 7 and `wait_completed` at 8" is a claim a reader can + * check against the output rather than translate first. + * + * The consequence worth knowing: the trace prints in *commit* order but is + * numbered in *log* order, so a run whose durable log disagrees with the order + * its writers actually committed in shows up as positions counting backwards. + * Those lines are highlighted. That disagreement is the entire subject of the + * red scenarios, and this is where you see it. + * + * Colour is decoration over that: it is applied only when the destination is a + * terminal, and is off under `NO_COLOR` or `--no-color`. With colour off the + * output is the same plain ASCII it always was, stable enough to check in as a + * golden file. + */ + +import type { Event } from '@workflow/world'; +import type { ScenarioResult } from './scenario.js'; +import type { TraceEntry, WriterId } from './types.js'; + +export interface RenderOptions { + /** Include `runs.get` / `events.list` style reads. Off by default: noisy. */ + verbose?: boolean; + /** Maximum characters of decoded payload to show per event. */ + payloadWidth?: number; + /** + * ANSI colour. Defaults to "on if stdout is a TTY and `NO_COLOR` is unset", + * so piping to a file or a golden-file comparison gets plain ASCII without + * anyone having to remember a flag. + */ + color?: boolean; +} + +const CHECK = 'ok'; +const CROSS = 'FAIL'; + +// --------------------------------------------------------------------------- +// Colour +// --------------------------------------------------------------------------- + +/** SGR codes, applied through `Paint` so a no-colour render never sees them. */ +const SGR = { + reset: 0, + bold: 1, + dim: 2, + red: 31, + green: 32, + yellow: 33, + blue: 34, + magenta: 35, + cyan: 36, + gray: 90, +} as const; + +type Style = keyof typeof SGR; + +/** + * A `paint(text, ...styles)` function that is either real or the identity. + * + * Resolving the on/off decision once, into a function, keeps every call site + * free of `if (color)` β€” which matters because getting one of them wrong is + * how escape codes leak into a file that was supposed to be diffable. + */ +type Paint = (text: string, ...styles: Style[]) => string; + +const plain: Paint = (text) => text; + +const ESC = '\u001b['; + +const ansi: Paint = (text, ...styles) => + styles.length === 0 + ? text + : `${ESC}${styles.map((s) => SGR[s]).join(';')}m${text}${ESC}${SGR.reset}m`; + +function colorEnabled(explicit: boolean | undefined): boolean { + if (explicit !== undefined) return explicit; + if (process.env.NO_COLOR) return false; + if (process.env.FORCE_COLOR) return true; + return Boolean(process.stdout?.isTTY); +} + +function painter(options: RenderOptions): Paint { + return colorEnabled(options.color) ? ansi : plain; +} + +/** + * Colour by event family, so a trace can be skimmed for shape before it is + * read for content: where the hooks are, where the waits are, and whether + * anything went red. + */ +function eventStyle(eventType: string): Style { + if (/(_failed|_cancelled|_conflict)$/.test(eventType)) return 'red'; + if (eventType.startsWith('run_')) return 'magenta'; + if (eventType.startsWith('step_')) return 'cyan'; + if (eventType.startsWith('hook_')) return 'green'; + if (eventType.startsWith('wait_')) return 'blue'; + if (eventType.startsWith('attr_')) return 'yellow'; + return 'gray'; +} + +// --------------------------------------------------------------------------- +// Event references +// --------------------------------------------------------------------------- + +/** + * Log positions for every event in a trace. + * + * Built by sorting the events the way `events.list` does, *not* by the order + * they were committed in β€” the two differ exactly when something interesting + * happened, and the number has to describe the durable log for a reader to be + * able to reason about what a replay will see. + */ +interface EventIndex { + /** Log position of an event, by its id. */ + position(eventId: string): number | undefined; + /** Log position at which a correlationId first appears. */ + origin(correlationId: string): number | undefined; + /** Width of the widest position, for column alignment. */ + width: number; +} + +function buildEventIndex(trace: readonly TraceEntry[]): EventIndex { + const events = trace + .filter((e): e is Extract => { + return e.kind === 'event'; + }) + .map((e) => e.event); + + const sorted = [...events].sort((a, b) => { + const at = a.createdAt.getTime(); + const bt = b.createdAt.getTime(); + if (at !== bt) return at - bt; + return a.eventId < b.eventId ? -1 : a.eventId > b.eventId ? 1 : 0; + }); + + const positions = new Map(); + const origins = new Map(); + sorted.forEach((event, i) => { + positions.set(event.eventId, i); + if (event.correlationId && !origins.has(event.correlationId)) { + origins.set(event.correlationId, i); + } + }); + + return { + position: (eventId) => positions.get(eventId), + origin: (correlationId) => origins.get(correlationId), + width: Math.max(2, String(Math.max(0, sorted.length - 1)).length), + }; +} + +/** `#12`, padded to the trace's position column. */ +function eventRef(position: number | undefined, width: number): string { + return position === undefined + ? `#${'?'.padStart(width)}` + : `#${String(position).padStart(width)}`; +} + +/** `@7` β€” the resource created at position 7. */ +function resourceRef(position: number | undefined): string { + return position === undefined ? '@?' : `@${position}`; +} + +/** + * Rewrite raw event ids in a message into `#N` log positions. + * + * Invariant and replay violations are written by code that only has ids to + * hand. Translating here rather than there keeps the one-scheme promise + * without making every check carry a positional index it has no other use + * for. + */ +function withRefs(message: string, index: EventIndex): string { + return message.replace(/\bevnt_[0-9A-Za-z]+\b/g, (id) => { + const position = index.position(id); + return position === undefined ? id : `#${position}`; + }); +} + +// --------------------------------------------------------------------------- +// Event detail +// --------------------------------------------------------------------------- + +function shortName(machineName: string | undefined): string | undefined { + if (!machineName) return undefined; + const parts = machineName.split('//'); + return parts[parts.length - 1] || machineName; +} + +/** One-line summary of what an event carries, beyond its type. */ +function describeEvent(event: Event, width: number): string { + const data = (event as { eventData?: Record }).eventData; + const bits: string[] = []; + + switch (event.eventType) { + case 'run_created': + case 'run_started': + if (data?.workflowName) + bits.push(String(shortName(String(data.workflowName)))); + break; + case 'step_created': + case 'step_started': + case 'step_completed': + case 'step_failed': + case 'step_retrying': + if (data?.stepName) bits.push(String(shortName(String(data.stepName)))); + if (typeof data?.attempt === 'number') + bits.push(`attempt=${data.attempt}`); + break; + case 'hook_created': + case 'hook_received': + case 'hook_disposed': + case 'hook_conflict': + if (data?.token) bits.push(`token=${JSON.stringify(data.token)}`); + break; + case 'wait_created': + case 'wait_completed': + if (data?.resumeAt) { + bits.push( + `resumeAt=${new Date(data.resumeAt as string).toISOString()}` + ); + } + break; + case 'attr_set': + bits.push( + (data?.changes as { key: string }[] | undefined) + ?.map((c) => c.key) + .join(',') ?? '' + ); + break; + default: + break; + } + + // Payload fields are devalue/CBOR blobs; showing their size is more honest + // than pretending to decode them here (the scenario result hydrates the run + // output properly, which is the payload that usually matters). + for (const field of [ + 'input', + 'result', + 'output', + 'payload', + 'error', + 'metadata', + ]) { + const value = data?.[field]; + if (value instanceof Uint8Array) { + bits.push(`${field}=<${value.byteLength}B>`); + } else if (value !== undefined) { + const text = JSON.stringify(value) ?? String(value); + bits.push( + `${field}=${text.length > width ? `${text.slice(0, width)}…` : text}` + ); + } + } + + return bits.filter(Boolean).join(' '); +} + +function offset(atMs: number, epochMs: number): string { + const delta = atMs - epochMs; + if (delta === 0) return '+0ms'; + if (delta < 1000) return `+${delta}ms`; + if (delta < 60_000) return `+${(delta / 1000).toFixed(1)}s`; + if (delta < 3_600_000) return `+${(delta / 60_000).toFixed(1)}m`; + if (delta < 86_400_000) return `+${(delta / 3_600_000).toFixed(1)}h`; + return `+${(delta / 86_400_000).toFixed(1)}d`; +} + +/** + * Short form of a writer id for the trace's fixed-width writer column. + * + * The column answers the question the log itself cannot: two adjacent events + * may have been committed by two different writers, and which one wrote which + * is exactly what an interleaving scenario is about. + */ +function shortWriter(writer: WriterId | undefined): string { + if (!writer) return ''; + if (writer === 'orchestrator') return 'wf'; + if (writer === 'external') return 'ext'; + return writer.startsWith('step:') ? writer.slice('step:'.length) : writer; +} + +export function renderTrace( + trace: readonly TraceEntry[], + options: RenderOptions = {} +): string { + const paint = painter(options); + const width = options.payloadWidth ?? 48; + const index = buildEventIndex(trace); + const epochMs = trace.length > 0 ? trace[0].atMs : 0; + // Sized to the widest writer actually seen, so the common case (a run with + // no steps, or short step names) does not pay for a name it never prints. + const writerWidth = trace.reduce( + (max, entry) => + entry.kind === 'event' || entry.kind === 'hold' + ? Math.max(max, shortWriter(entry.writer).length) + : max, + 2 + ); + const blank = ' '.repeat(index.width + 1); + const lines: string[] = []; + // Highest log position printed so far. A line below it is an event that was + // committed after one that outranks it in the log β€” the disagreement the six + // red scenarios are about. + let highWater = -1; + + for (const entry of trace) { + const time = paint(offset(entry.atMs, epochMs).padStart(8), 'dim'); + // A depth > 0 entry happened inside a script action, i.e. inside another + // world call. Indenting it is the visual proof of the ordering the + // scenario asked for. + const indent = ' '.repeat(entry.depth); + const writerName = + entry.kind === 'event' || entry.kind === 'hold' + ? shortWriter(entry.writer) + : ''; + // Pad before painting: `padEnd` counts escape bytes, so a coloured column + // padded afterwards comes out short by however long the escape is. + const writer = + entry.kind === 'event' || entry.kind === 'hold' + ? `${paint( + writerName.padEnd(writerWidth), + writerName === 'ext' ? 'magenta' : 'dim' + )} ` + : `${' '.repeat(writerWidth)} `; + + switch (entry.kind) { + case 'event': { + const position = index.position(entry.event.eventId); + const backwards = position !== undefined && position < highWater; + if (position !== undefined && position > highWater) + highWater = position; + + const ref = eventRef(position, index.width); + const type = entry.event.eventType; + const correlation = entry.event.correlationId + ? ` ${paint( + resourceRef(index.origin(entry.event.correlationId)), + 'gray' + )}` + : ''; + const detail = describeEvent(entry.event, width); + + lines.push( + `${paint(ref, ...(backwards ? (['yellow', 'bold'] as const) : (['dim'] as const)))} ${time} ` + + `${writer}${indent}${paint(type.padEnd(16), eventStyle(type))}` + + `${correlation}${detail ? ` ${paint(detail, 'dim')}` : ''}` + ); + break; + } + case 'hold': + lines.push( + `${blank} ${time} ${writer}${indent}${paint(`>> held "${entry.label}" at ${entry.inside}`, 'yellow')}` + ); + break; + case 'delivery': + if (!options.verbose) break; + lines.push( + `${blank} ${time} ${writer}${indent}${paint(`-- ${entry.message}`, 'gray')}` + ); + break; + case 'note': + lines.push( + `${blank} ${time} ${writer}${indent}${paint(`// ${entry.message}`, 'dim')}` + ); + break; + case 'warn': + lines.push( + `${blank} ${time} ${writer}${indent}${paint(`!! ${withRefs(entry.message, index)}`, 'red')}` + ); + break; + case 'check': + lines.push( + `${blank} ${time} ${writer}${indent}${paint( + `${entry.ok ? CHECK : CROSS} check: ${entry.name}`, + entry.ok ? 'green' : 'red' + )}` + ); + break; + } + } + + return lines.join('\n'); +} + +export function renderScenario( + result: ScenarioResult, + options: RenderOptions = {} +): string { + const paint = painter(options); + const out: string[] = []; + const status = result.ok + ? paint('PASS', 'green', 'bold') + : paint('FAIL', 'red', 'bold'); + + out.push(`${status} ${paint(result.id, 'bold')}`); + out.push(` ${result.name}`); + if (result.description) out.push(` ${paint(result.description, 'dim')}`); + out.push( + paint( + ` run=${result.runId || '(none)'} outcome=${result.outcome} ` + + `events=${result.events.length} deliveries=${result.deliveries} ` + + `worldCalls=${result.worldCalls} virtual=${formatDuration(result.virtualElapsedMs)} ` + + `wall=${result.wallMs.toFixed(0)}ms replay=${describeReplay(result)}` + + // Only when it is on. The default is production, and a line that + // repeats "this is the ordinary world" on every scenario says nothing. + (result.appendOnlyLog ? ' log=append-only' : ''), + 'dim' + ) + ); + out.push(''); + out.push(renderTrace(result.trace, options)); + + const index = buildEventIndex(result.trace); + + if (result.output !== undefined) { + out.push(''); + out.push(` output: ${safeJson(result.output)}`); + } + + const attributes = result.run?.attributes ?? {}; + if (Object.keys(attributes).length > 0) { + out.push(''); + out.push(` attributes: ${safeJson(attributes)}`); + } + + if (result.pending.length > 0) { + out.push(''); + out.push(` ${result.pending.length} message(s) still queued:`); + for (const m of result.pending) { + out.push( + ` ${m.messageId} ready+${m.readyAtMs} deliveries=${m.deliveries}` + ); + } + } + + if (result.violations.length > 0) { + out.push(''); + out.push(paint(' CONSISTENCY VIOLATIONS', 'red', 'bold')); + for (const v of result.violations) { + const at = + v.eventId && index.position(v.eventId) !== undefined + ? ` (at #${index.position(v.eventId)})` + : ''; + out.push( + ` ${paint(`[${v.rule}]`, 'red')} ${withRefs(v.message, index)}${paint(at, 'dim')}` + ); + } + } + + if (result.problems.length > 0) { + out.push(''); + out.push(paint(' PROBLEMS', 'red', 'bold')); + for (const p of result.problems) out.push(` ${withRefs(p, index)}`); + } + + return out.join('\n'); +} + +export function renderSummary( + results: readonly ScenarioResult[], + options: RenderOptions = {} +): string { + const paint = painter(options); + const passed = results.filter((r) => r.ok).length; + const failed = results.length - passed; + const violations = results.reduce((n, r) => n + r.violations.length, 0); + return [ + '', + `${results.length} scenario(s): ` + + `${paint(`${passed} passed`, 'green')}, ` + + `${paint(`${failed} failed`, failed > 0 ? 'red' : 'dim')}, ` + + `${paint(`${violations} consistency violation(s)`, violations > 0 ? 'red' : 'dim')}`, + ].join('\n'); +} + +export interface MarkdownSummaryOptions { + /** Label on the fold's visible line. Defaults to `world-sim`. */ + title?: string; + /** + * Which world produced these results, as short `key=value` chips above the + * table β€” `log=append-only`, `fence=off`. A summary that does not say which + * world it ran in is unreadable next to another one, and the whole point of + * this book is comparing two runs of it. + */ + chips?: readonly string[]; + /** + * Where the full trace was written, mentioned under the table so a reader + * who needs more than a row knows an artifact exists. + */ + detailPath?: string; +} + +/** + * The same counts as `renderSummary`, as GitHub-flavoured markdown sized for a + * PR comment or `$GITHUB_STEP_SUMMARY`. + * + * One collapsed `
`: a visible line carrying the count and a green or + * orange dot, and the whole table behind it. Built to be stacked β€” a CI job + * plays the book once per world and puts two of these under one heading β€” so + * it renders no heading of its own, and nothing above the fold but the count. + * + * There is deliberately no list of failures. Six of them are red on purpose, + * so a comment that leads with the failures leads with the part that is not + * news, and it grows a wall of text on exactly the PRs that changed nothing. + * The count is the signal; the names are one click away. + * + * Never coloured β€” ANSI in a markdown file renders as garbage. + */ +export function renderMarkdownSummary( + results: readonly ScenarioResult[], + options: MarkdownSummaryOptions = {} +): string { + const failed = results.filter((r) => !r.ok).length; + const out: string[] = []; + + // A dot rather than words: `` is one line of a collapsed comment, + // and markdown has no colour, so this is the only way the two worlds read as + // different at a glance without being read at all. + out.push('
'); + out.push( + `${failed > 0 ? '🟠' : '🟒'} ${options.title ?? 'world-sim'}` + + ` β€” ${failed} fail of ${results.length} total` + ); + // Blank line after `
`, or GitHub renders the table as literal + // pipes. + out.push(''); + + if (options.chips && options.chips.length > 0) { + out.push(options.chips.map((c) => `\`${c}\``).join(' Β· ')); + out.push(''); + } + + out.push('| scenario | outcome | events | virt | replay | violations |'); + out.push('| --- | --- | --- | --- | --- | --- |'); + for (const r of results) { + out.push( + `| ${r.ok ? 'βœ…' : '❌'} \`${r.id}\` | ${r.outcome} | ${r.events.length} | ` + + `${formatDuration(r.virtualElapsedMs)} | ${describeReplay(r)} | ` + + `${r.violations.length} |` + ); + } + + if (options.detailPath) { + out.push(''); + out.push(`Full trace: \`${options.detailPath}\``); + } + out.push(''); + out.push('
'); + out.push(''); + return out.join('\n'); +} + +/** Short form of the cold-replay check for the summary line. */ +function describeReplay(result: ScenarioResult): string { + if (!result.replay) return 'not-run'; + if ('skipped' in result.replay) return 'skipped'; + const failed = result.violations.some((v) => v.rule.startsWith('replay.')); + return failed ? 'MISMATCH' : 'ok'; +} + +function formatDuration(ms: number): string { + if (ms < 1000) return `${ms}ms`; + if (ms < 60_000) return `${(ms / 1000).toFixed(1)}s`; + if (ms < 3_600_000) return `${(ms / 60_000).toFixed(1)}m`; + if (ms < 86_400_000) return `${(ms / 3_600_000).toFixed(1)}h`; + return `${(ms / 86_400_000).toFixed(1)}d`; +} + +function safeJson(value: unknown): string { + try { + return JSON.stringify(value) ?? String(value); + } catch { + return String(value); + } +} diff --git a/packages/world-sim/src/scenario.ts b/packages/world-sim/src/scenario.ts new file mode 100644 index 0000000000..10a841ccf2 --- /dev/null +++ b/packages/world-sim/src/scenario.ts @@ -0,0 +1,704 @@ +/** + * Scenario definition and the deterministic scheduler that plays one out. + * + * A scenario is one workflow, its input, and a script that steers the run's + * writers. Playing it consists of exactly one loop: take the next queue + * message, jump the virtual clock to its delivery time, hand it to the flow + * handler, repeat until the queue is empty. Nothing else can advance the world + * β€” no timers, no background delivery, no wall-clock waiting β€” so the sequence + * of world calls is reproducible. + * + * Termination is a hard requirement, and three separate things enforce it: + * + * - **Virtual time.** A `sleep('30d')` is a queue message dated 30 days out; + * delivering it means moving a number, not waiting. + * - **Quiescence.** An empty queue ends the loop. If the run has not reached + * a terminal state at that point the scenario is *stalled* β€” reported, with + * the open hooks and waits that explain it, rather than hung. + * - **Budgets.** Delivery count, virtual span and wall time are all capped, + * so even a workflow that genuinely loops forever ends as a failed scenario + * instead of a wedged process. + */ + +import { setTimeout as sleep } from 'node:timers/promises'; +import { resumeHook, setWorld, start } from '@workflow/core/runtime'; +import { + type Event, + SPEC_VERSION_CURRENT, + type WorkflowRun, + type WorkflowRunStatus, +} from '@workflow/world'; +import { createVirtualClock } from './clock.js'; +import { + DEFAULT_LIMITS, + deliver, + driveQueue, + performanceNow, + type ScenarioLimits, + type SelectNext, +} from './drive.js'; +import { checkInvariants } from './invariants.js'; +import { verifyReplay } from './replay.js'; +import { createTempo, ScenarioAborted } from './tempo.js'; +import type { + InvariantViolation, + PendingMessageView, + ScenarioApi, + ScenarioScript, + TraceEntry, +} from './types.js'; +import { + createSimWorld, + type SimWorld, + WORKFLOW_QUEUE_PREFIX, +} from './world.js'; + +/** Real-time grace period for a script to unwind after the scenario aborts it. */ +const SCRIPT_UNWIND_MS = 50; + +export interface ScenarioSpec { + /** + * Stable hyphenated handle for this scenario, e.g. `hook-at-step-started`. + * + * The `name` is prose and will be reworded; the id is what a commit message, + * a bug report or a `pnpm sim ` invocation refers to, so it is expected + * to outlive several rewordings of the sentence next to it. + */ + id: string; + name: string; + description?: string; + /** + * The workflow to run: either its plain function name (resolved against the + * build manifest) or an explicit `{ workflowId }`. + */ + workflow: string | { workflowId: string }; + /** Arguments passed to the workflow function. */ + input?: unknown[]; + /** + * When external input arrives and how the run's writers interleave, written + * as a sequence of writer advances. Runs concurrently with the delivery loop + * and starts before the run does, so it can hold the very first world call. + * + * A scenario with no script is a control: the run plays out on the default + * schedule and the only question is whether the log it leaves reproduces it. + */ + script?: ScenarioScript; + /** + * Override which queue message is delivered next. + * + * The default is total and deterministic β€” earliest `readyAt`, then enqueue + * order β€” which is the right model for a queue whose delays are real + * deadlines. Override it to pin an order the default would not produce, e.g. + * to deliver a later message first and check the run tolerates it. Return a + * `messageId` from `pending`, or `undefined` to fall back to the default. + */ + selectNext?: SelectNext; + /** + * Cold-replay the committed log at the end and check it regenerates the run. + * On by default for runs that reached `completed` or `failed`; see + * `verifyReplay`. + */ + verifyReplay?: boolean; + /** + * Assertions about how the run should end β€” what *correct* looks like, which + * is not always what happens today. + * + * `status` accepts the non-run outcomes too (`stalled`, `budget-exceeded`), + * because "this workflow deadlocks when the hook never arrives" is a + * property worth pinning down rather than an accident to be tolerated. + * + * There is deliberately no way to expect a consistency violation. A scenario + * that reproduces a corruption states the outcome the run *should* have had + * and fails until the runtime delivers it: the failure is the open bug, and + * it goes green when the bug is fixed, not when it is observed one more time. + * Any violation fails the scenario that tripped it. + * + * There is also deliberately no per-world variant of this field. A scenario + * is one sequence of advances, and the only thing a world changes is what a + * read returns β€” so an expectation that has to be restated per world is + * pinning a consequence of the reads rather than a property of the run. Pin + * the part that holds in every world; where the branch a run takes is decided + * by what it read, do not pin the branch. What catches the fault there is the + * invariant, not the expectation: the log a run wrote must be a log the + * runtime can replay back into that same run, and that sentence is true in + * both worlds. `verifyReplay` is on by default for exactly this reason, and + * every red in the book is red on the invariant alone β€” the expectations + * could all be deleted without changing which scenarios fail. + */ + expect?: ScenarioExpectation; + limits?: ScenarioLimits; + /** Enforce (and advertise) the optimistic-concurrency fence. */ + preconditionGuard?: boolean; + /** + * Also enforce the count half of the fence: reject a write whose caller loaded + * fewer events at or below its watermark than the log now holds. + * + * Defaults to `preconditionGuard`, because that is production: since #3145 + * `@workflow/core` sends `stateEventCount` on every replay-context create and + * workflow-server's count guard is on by default, so a fence is a fence with + * both halves. Set it to `false` alongside `preconditionGuard: true` to model + * the watermark alone. + */ + countGuard?: boolean; + /** + * Run this scenario against an append-only log, where an event takes its + * position at commit and a read can therefore be behind but never + * self-contradictory. See `SimStoreOptions.appendOnlyLog`. + * + * Off by default, and no scenario in the book sets it: the interesting use is + * as a global override (`RunScenarioOptions.appendOnlyLog`), where the whole + * book runs twice and the diff is the answer to "which of these failures are + * about the mint-before-commit window, and which are about something else?". + */ + appendOnlyLog?: boolean; +} + +export interface ScenarioExpectation { + status?: ScenarioOutcome; + /** Compared against the hydrated run output with deep equality. */ + output?: unknown; +} + +export type ScenarioOutcome = + | WorkflowRunStatus + | 'stalled' + | 'budget-exceeded' + | 'error'; + +export interface ScenarioResult { + /** The spec's stable handle. See `ScenarioSpec.id`. */ + id: string; + name: string; + description?: string; + runId: string; + outcome: ScenarioOutcome; + ok: boolean; + /** + * Which log the run played against β€” resolved from the spec and the run + * option, so a stored result says which world produced it rather than + * leaving the reader to remember. + */ + appendOnlyLog: boolean; + /** Scenario-level failures: unmet expectations, failed checks, script errors. */ + problems: string[]; + /** World-contract violations found by re-deriving state from the log. */ + violations: InvariantViolation[]; + /** + * Outcome of the cold-replay check, when it ran. `skipped` carries why β€” + * a cancelled or stalled run has no workflow-derived terminal event for a + * replay to re-derive. + */ + replay?: { deliveries: number; regenerated: number } | { skipped: string }; + events: Event[]; + trace: TraceEntry[]; + run?: WorkflowRun; + /** Hydrated run output, when the run completed and hydration succeeded. */ + output?: unknown; + /** Anything still queued when the loop ended. */ + pending: PendingMessageView[]; + deliveries: number; + worldCalls: number; + virtualElapsedMs: number; + wallMs: number; + /** Populated when the runner itself threw. */ + error?: unknown; +} + +export interface RunScenarioOptions { + /** The compiled flow handler (see `loadFlowHandler`). */ + handler: (req: Request) => Promise; + /** Workflow function name β†’ machine workflow id, from `buildSimBundle`. */ + workflowIds?: Record; + /** + * Force `SimStoreOptions.appendOnlyLog` on or off for this run, overriding + * whatever the spec asked for. + * + * This is the knob a caller flips to play the same book under both worlds β€” + * the CLI's `--append-only`, the page's toggle. `undefined` leaves the + * decision to the spec, which is how a book of scenarios written against + * production keeps behaving like one. + */ + appendOnlyLog?: boolean; + /** + * Force `SimStoreOptions.preconditionGuard` on or off for this run, + * overriding whatever the spec asked for. + * + * The fence exists to reject a write whose snapshot predates an out-of-band + * event β€” that is, a write an extended prefix invalidated. So turning it off + * across the whole book answers a question the book cannot otherwise ask: is + * any scenario relying on it? If none is, then no emitter here is + * prefix-sensitive and the fence guards against nothing; if one goes red that + * was not red before, that scenario names the exception. + * + * Disabling it takes the count half with it: `countGuard` is evaluated inside + * the same predicate, and the marker bookkeeping that both halves read is + * gated on the same flag. + */ + preconditionGuard?: boolean; +} + +export async function runScenario( + spec: ScenarioSpec, + options: RunScenarioOptions +): Promise { + const limits = { ...DEFAULT_LIMITS, ...spec.limits }; + const clock = createVirtualClock(); + const appendOnlyLog = options.appendOnlyLog ?? spec.appendOnlyLog ?? false; + // One expectation, whichever world this is. Aliased rather than read inline + // so the outcome check and the output check below cannot drift apart. + const expected = spec.expect; + // Production arms both halves of the fence, so the count follows it unless a + // scenario says otherwise. Resolved once: reading `options ?? spec` twice + // would let a `--no-fence` run keep a count guard the fence no longer backs. + const preconditionGuard = + options.preconditionGuard ?? spec.preconditionGuard ?? false; + const world = createSimWorld({ + clock, + preconditionGuard, + countGuard: spec.countGuard ?? preconditionGuard, + appendOnlyLog, + }); + + const workflowId = + typeof spec.workflow === 'string' + ? options.workflowIds?.[spec.workflow] + : spec.workflow.workflowId; + + if (!workflowId) { + return failedBeforeStart( + spec, + `Unknown workflow "${String(spec.workflow)}". Known: ${Object.keys( + options.workflowIds ?? {} + ) + .filter((k) => !k.includes('#')) + .sort() + .join(', ')}`, + appendOnlyLog + ); + } + + const problems: string[] = []; + /** + * Problems that are only problems if the scenario did not ask for them. A + * scenario may legitimately assert that a run stalls or blows its budget β€” + * those are properties worth pinning down β€” so the diagnosis is always + * recorded but only counted as a failure when it was a surprise. + */ + const outcomeProblems: string[] = []; + let replayViolations: InvariantViolation[] = []; + let replay: ScenarioResult['replay']; + let virtualElapsedMs = 0; + let deliveries = 0; + let outcome: ScenarioOutcome | undefined; + let runId = ''; + let thrown: unknown; + + let uninstallClock = clock.install(); + const wallStart = performanceNow(); + + world.registerHandler(WORKFLOW_QUEUE_PREFIX, options.handler); + + const api: ScenarioApi = { + world: world.snapshot, + appendOnlyLog, + get runId() { + return runId; + }, + async deliverHook(token, payload) { + // Go through the real `resumeHook` rather than writing `hook_received` + // directly: the point of the simulation is to exercise the same code an + // out-of-band webhook receiver would run, including its payload + // dehydration, its terminal-run rejection mapping, and its re-enqueue. + // + // `asExternal` marks the writes as the scenario's own: they are attributed + // to the `external` writer and are not themselves call points, so a + // script that delivers a hook while holding a writer cannot trip one of + // its own holds. + await world.asExternal(() => resumeHook(token, payload)); + }, + async beginHookDelivery(token, payload) { + // Take the position now and commit later. This is the handler boundary + // split made visible to a script: the receiver has entered `resumeHook`, + // its event id is minted and its slot in the log is spoken for, but the + // storage write has not happened β€” so every other writer that commits in + // the meantime lands *behind* a position nobody can see yet. + // + // It has to be an out-of-band writer. An inline step's `step_completed` + // held the same way would stall the orchestrator that should misread the + // log, because the runtime awaits its promise before deciding anything. + const position = world.reservePosition(); + return { + eventId: position.eventId, + async commit() { + await world.asExternal(() => + world.withReservedPosition(position, () => + resumeHook(token, payload) + ) + ); + }, + }; + }, + async cancelRun(reason) { + await world.asExternal(() => + world.events.create(runId, { + eventType: 'run_cancelled', + specVersion: SPEC_VERSION_CURRENT, + ...(reason ? { eventData: { cancelReason: reason } } : {}), + }) + ); + }, + withholdNextEvent(reads) { + world.store.withholdNextEvent(reads); + }, + advanceTime(ms) { + clock.advanceBy(ms); + world.pushTrace({ + kind: 'note', + message: `advanced virtual time by ${ms}ms`, + }); + }, + async deliverQueued(select) { + const pending = world.simQueue.view(); + const chosen = select ? select(pending) : pending[0]?.messageId; + if (!chosen) return false; + // `takeById` removes it from pending, so the delivery loop cannot pick + // the same message up: the two never race for one message, they only run + // two different ones at once. + const message = world.simQueue.takeById(chosen); + if (!message) return false; + clock.advanceTo(message.readyAtMs); + await deliver(world, message); + return true; + }, + note(message) { + world.pushTrace({ kind: 'note', message }); + }, + check(name, condition) { + world.pushTrace({ kind: 'check', name, ok: condition }); + if (!condition) problems.push(`check failed: ${name}`); + }, + }; + world.setScenarioApi(() => api); + + // The script steers writers by watching world calls, so its machinery must + // exist before anything can fire β€” and it is launched before `start()` so it + // can hold the very first world call the run makes. + const controller = createTempo(world, api, { + runToWallMs: Math.min(limits.maxRunToWallMs, limits.maxWallMs), + }); + let scriptSettled = spec.script === undefined; + let scriptError: unknown; + const scriptDone = spec.script + ? (async () => spec.script?.(controller.tempo))() + .catch((err) => { + scriptError = err; + }) + .finally(() => { + scriptSettled = true; + }) + : undefined; + + /** + * Real-time backstop. A parked call blocks the scheduler, so a script that + * waits for something that never happens is a hang, not a stall β€” the one + * way to lose the termination guarantee. This buys it back, and it has to be + * wall-clock: the virtual clock is precisely what stops advancing. + * + * It is armed slightly past the loop's own wall budget so that whenever the + * loop is able to notice the overrun itself, it reports it with the better + * diagnosis; this fires only when the loop is blocked inside a held call. + */ + const deadline = setTimeout(() => { + const reason = + `scenario exceeded its ${limits.maxWallMs}ms wall-clock budget while the ` + + 'script held or awaited a world call'; + world.pushTrace({ kind: 'warn', message: reason }); + problems.push(reason); + controller.abort(reason); + }, limits.maxWallMs + 250); + // Deliberately NOT unref'd. A total deadlock β€” every writer parked, the + // scheduler blocked inside a held call, the script awaiting something that + // can never happen β€” leaves nothing else on the event loop. An unref'd + // watchdog does not hold the loop open, so Node would empty it and exit with + // a bare "unsettled top-level await" instead of this timer firing: the + // watchdog would be absent from the one case it exists for. The `finally` + // below clears it, so keeping it ref'd cannot outlive the scenario. + + setWorld(world); + + try { + const run = await start({ workflowId }, (spec.input ?? []) as unknown[]); + runId = run.runId; + + const drain = await driveQueue({ + world, + limits, + wallStart, + selectNext: spec.selectNext, + }); + deliveries = drain.deliveries; + if (drain.exceeded) { + outcome = 'budget-exceeded'; + outcomeProblems.push(drain.exceeded); + } + + // ---- Replay verification --------------------------------------------- + // Snapshot the run's own virtual span first: the replay reuses the clock + // (it has to β€” `Date.now()` is what tells the runtime a wait has elapsed) + // and would otherwise show up in the scenario's reported timings. + virtualElapsedMs = clock.elapsed(); + const finished = world.store.allRuns().find((r) => r.runId === runId); + if (spec.verifyReplay === false) { + replay = { skipped: 'disabled for this scenario' }; + } else if (!finished) { + replay = { skipped: 'no run entity' }; + } else if ( + finished.status !== 'completed' && + finished.status !== 'failed' + ) { + // A cancelled run's terminal event came from an operator, not from the + // workflow, and a stalled run has none at all β€” in neither case is there + // a workflow-derived answer for a replay to reproduce. + replay = { + skipped: `run ended "${finished.status}", which the workflow did not derive`, + }; + } else { + // `verifyReplay` installs a clock of its own, pinned to the instant the + // run ended, so this one has to step aside for the duration. + uninstallClock(); + let check: Awaited>; + try { + check = await verifyReplay({ + run: finished, + events: world.store.allEvents(runId), + handler: options.handler, + limits, + appendOnlyLog, + }); + } finally { + uninstallClock = clock.install(); + setWorld(world); + } + replayViolations = check.violations; + replay = { + deliveries: check.deliveries, + regenerated: check.regenerated.length, + }; + world.pushTrace({ + kind: 'note', + message: + `replay check: cold-started from the committed log in ${check.deliveries} ` + + `delivery(ies), re-derived ${check.regenerated.length} event(s)` + + (check.violations.length === 0 ? ' β€” matches' : ' β€” MISMATCH'), + }); + } + } catch (err) { + thrown = err; + outcome = 'error'; + problems.push( + `scenario threw: ${err instanceof Error ? err.message : String(err)}` + ); + } finally { + // The loop is done, so nothing can satisfy a script still waiting on the + // world. Report what it wanted before tearing its waits down. + const stillWaiting = controller.describeWaiting(); + if (!scriptSettled && stillWaiting) { + problems.push(`script never finished β€” ${stillWaiting}`); + } + controller.abort('scenario ended'); + clearTimeout(deadline); + + // Give the script a bounded moment to unwind from the rejections `abort` + // just raised into it. Awaiting it outright would reintroduce the hang + // this whole mechanism exists to prevent: a script parked on a promise + // that never settles (rather than on the world) is unreachable from here. + await Promise.race([scriptDone, sleep(SCRIPT_UNWIND_MS)]); + if (!scriptSettled && !stillWaiting) { + problems.push( + 'script never finished β€” it is blocked on something outside the world' + ); + } + if ( + scriptError !== undefined && + !(scriptError instanceof ScenarioAborted) + ) { + problems.push( + `script threw: ${ + scriptError instanceof Error + ? scriptError.message + : String(scriptError) + }` + ); + } + world.streamer.abortOpenReaders(); + setWorld(undefined); + uninstallClock(); + } + + const wallMs = performanceNow() - wallStart; + const events = runId ? world.store.allEvents(runId) : []; + const runEntity = runId + ? world.store.allRuns().find((r) => r.runId === runId) + : undefined; + + if (!outcome) { + if ( + runEntity && + runEntity.status !== 'pending' && + runEntity.status !== 'running' + ) { + outcome = runEntity.status; + } else { + outcome = 'stalled'; + const reason = describeStall(world, runId); + world.pushTrace({ kind: 'note', message: `STALLED: ${reason}` }); + outcomeProblems.push(reason); + } + } + + if (expected?.status !== outcome) problems.push(...outcomeProblems); + + const violations = runId + ? checkInvariants({ + runId, + events, + // Only meaningful when the world promises commit order is log order. + ...(appendOnlyLog + ? { eventsInCommitOrder: world.store.allEventsInCommitOrder(runId) } + : {}), + runs: world.store.allRuns(), + steps: world.store.allSteps(runId), + waits: world.store.allWaits(runId), + }).concat(replayViolations) + : []; + + problems.push(...world.watchErrors()); + + let output: unknown; + if (runEntity?.status === 'completed' && runEntity.output !== undefined) { + output = await hydrateOutput(runEntity.output); + } + + if (expected?.status && outcome !== expected.status) { + problems.push(`expected run to end "${expected.status}", got "${outcome}"`); + } + if (expected && 'output' in expected) { + if (!deepEqual(output, expected.output)) { + problems.push( + `expected output ${JSON.stringify(expected.output)}, got ${JSON.stringify(output)}` + ); + } + } + + return { + id: spec.id, + name: spec.name, + description: spec.description, + runId, + outcome, + ok: problems.length === 0 && violations.length === 0 && outcome !== 'error', + appendOnlyLog, + problems, + violations, + events, + trace: world.trace, + run: runEntity, + output, + pending: world.simQueue.view(), + replay, + deliveries, + worldCalls: world.callCount(), + virtualElapsedMs, + wallMs, + error: thrown, + }; +} + +function describeStall(world: SimWorld, runId: string): string { + const events = runId ? world.store.allEvents(runId) : []; + const receivedHookIds = new Set( + events + .filter((e) => e.eventType === 'hook_received') + .map((e) => e.correlationId) + ); + const openHooks = world.store + .allHooks(runId) + .filter((h) => !receivedHookIds.has(h.hookId)); + const openWaits = world.store + .allWaits(runId) + .filter((w) => w.status === 'waiting'); + + const parts: string[] = [ + 'run never reached a terminal state and the world went quiet', + ]; + if (openHooks.length > 0) { + parts.push( + `waiting on ${openHooks.length} hook(s) that were never delivered: ${openHooks + .map((h) => JSON.stringify(h.token)) + .join(', ')}` + ); + } + if (openWaits.length > 0) { + parts.push( + `${openWaits.length} wait(s) still open with no continuation queued (this is a runtime bug β€” a pending wait should always have a queued continuation)` + ); + } + if (openHooks.length === 0 && openWaits.length === 0) { + parts.push( + 'no open hooks or waits β€” the workflow suspended with nothing to wake it' + ); + } + return parts.join('; '); +} + +async function hydrateOutput(raw: unknown): Promise { + const { hydrateData, observabilityRevivers } = await import( + '@workflow/core/serialization-format' + ); + try { + return hydrateData(raw, observabilityRevivers); + } catch { + return raw; + } +} + +function deepEqual(a: unknown, b: unknown): boolean { + if (Object.is(a, b)) return true; + if (typeof a !== typeof b || a === null || b === null) return false; + if (typeof a !== 'object') return false; + if (Array.isArray(a) !== Array.isArray(b)) return false; + const ak = Object.keys(a as object); + const bk = Object.keys(b as object); + if (ak.length !== bk.length) return false; + return ak.every((k) => + deepEqual( + (a as Record)[k], + (b as Record)[k] + ) + ); +} + +function failedBeforeStart( + spec: ScenarioSpec, + problem: string, + appendOnlyLog: boolean +): ScenarioResult { + return { + id: spec.id, + name: spec.name, + description: spec.description, + runId: '', + outcome: 'error', + ok: false, + appendOnlyLog, + problems: [problem], + violations: [], + events: [], + trace: [], + pending: [], + deliveries: 0, + worldCalls: 0, + virtualElapsedMs: 0, + wallMs: 0, + }; +} diff --git a/packages/world-sim/src/store.test.ts b/packages/world-sim/src/store.test.ts new file mode 100644 index 0000000000..f72a987b04 --- /dev/null +++ b/packages/world-sim/src/store.test.ts @@ -0,0 +1,764 @@ +import { + EntityConflictError, + HookNotFoundError, + RunExpiredError, +} from '@workflow/errors'; +import { SPEC_VERSION_CURRENT } from '@workflow/world'; +import { beforeEach, describe, expect, it } from 'vitest'; +import { createIdFactory } from './ids.js'; +import { + createSimStore, + type MintedEvent, + type SimStore, + type SimStoreOptions, + type StaleRead, +} from './store.js'; + +const SPEC = SPEC_VERSION_CURRENT; + +function setup(options?: Omit) { + let now = 1_704_067_200_000; + const store = createSimStore({ + now: () => now, + ids: createIdFactory(() => now), + ...options, + }); + return { store, tick: (ms: number) => (now += ms), nowMs: () => now }; +} + +async function createRun(store: SimStore, runId: string) { + await store.events.create(runId, { + eventType: 'run_created', + specVersion: SPEC, + eventData: { + deploymentId: 'dpl_sim', + workflowName: 'workflow//./w//demo', + input: new Uint8Array([1]), + }, + }); + await store.events.create(runId, { + eventType: 'run_started', + specVersion: SPEC, + }); +} + +type CreateParams = Parameters[2]; + +/** + * Commit at a position minted earlier β€” the hold the world facade opens + * between the handler boundary and the storage write. `minted` rides on the + * store's internal create params, which are deliberately not public, so a test + * driving the store directly has to say so out loud. + */ +function heldAt(minted: MintedEvent): CreateParams { + return { minted } as unknown as CreateParams; +} + +const RUN = 'wrun_01HK153X00000000000105JM0S'; + +describe('sim store', () => { + let store: SimStore; + let tick: (ms: number) => void; + + beforeEach(() => { + ({ store, tick } = setup()); + }); + + it('materializes the run entity from run lifecycle events', async () => { + await createRun(store, RUN); + const running = await store.runs.get(RUN); + expect(running.status).toBe('running'); + expect(running.startedAt).toBeInstanceOf(Date); + + await store.events.create(RUN, { + eventType: 'run_completed', + specVersion: SPEC, + eventData: { output: new Uint8Array([9]) }, + }); + const done = await store.runs.get(RUN); + expect(done.status).toBe('completed'); + expect(done.output).toEqual(new Uint8Array([9])); + }); + + it('makes run_started idempotent without duplicating the event', async () => { + await createRun(store, RUN); + const before = store.allEvents(RUN).length; + const result = await store.events.create(RUN, { + eventType: 'run_started', + specVersion: SPEC, + }); + expect(result.event).toBeUndefined(); + expect(store.allEvents(RUN).length).toBe(before); + }); + + it('bootstraps a run from run_started when run_created never landed', async () => { + // The resilient-start path: `start()` fires run_created and the queue + // message concurrently, and the queue message is allowed to win. + await store.events.create(RUN, { + eventType: 'run_started', + specVersion: SPEC, + eventData: { + deploymentId: 'dpl_sim', + workflowName: 'workflow//./w//demo', + input: new Uint8Array([1]), + }, + }); + const events = store.allEvents(RUN); + expect(events.map((e) => e.eventType)).toEqual([ + 'run_created', + 'run_started', + ]); + // The synthetic run_created must sort first, or replay sees the run start + // before it exists. + expect(events[0].eventId < events[1].eventId).toBe(true); + expect((await store.runs.get(RUN)).status).toBe('running'); + }); + + describe('step lifecycle', () => { + beforeEach(() => createRun(store, RUN)); + + it('tracks attempts across retries', async () => { + await store.events.create(RUN, { + eventType: 'step_created', + specVersion: SPEC, + correlationId: 'step_1', + eventData: { stepName: 'step//./w//charge', input: new Uint8Array() }, + }); + await store.events.create(RUN, { + eventType: 'step_started', + specVersion: SPEC, + correlationId: 'step_1', + }); + await store.events.create(RUN, { + eventType: 'step_retrying', + specVersion: SPEC, + correlationId: 'step_1', + eventData: { error: new Uint8Array() }, + }); + await store.events.create(RUN, { + eventType: 'step_started', + specVersion: SPEC, + correlationId: 'step_1', + }); + const step = await store.steps.get(RUN, 'step_1'); + expect(step.attempt).toBe(2); + expect(step.status).toBe('running'); + }); + + it('rejects a second terminal write for the same step', async () => { + await store.events.create(RUN, { + eventType: 'step_created', + specVersion: SPEC, + correlationId: 'step_1', + eventData: { stepName: 'step//./w//charge', input: new Uint8Array() }, + }); + await store.events.create(RUN, { + eventType: 'step_started', + specVersion: SPEC, + correlationId: 'step_1', + }); + await store.events.create(RUN, { + eventType: 'step_completed', + specVersion: SPEC, + correlationId: 'step_1', + eventData: { result: new Uint8Array() }, + }); + await expect( + store.events.create(RUN, { + eventType: 'step_completed', + specVersion: SPEC, + correlationId: 'step_1', + eventData: { result: new Uint8Array() }, + }) + ).rejects.toBeInstanceOf(EntityConflictError); + }); + + it('creates the step on a lazy step_started and keeps the log sorted', async () => { + const result = await store.events.create(RUN, { + eventType: 'step_started', + specVersion: SPEC, + correlationId: 'step_1', + eventData: { + stepName: 'step//./w//charge', + input: new Uint8Array([7]), + }, + }); + expect(result.stepCreated).toBe(true); + + const events = store.allEvents(RUN).slice(-2); + expect(events.map((e) => e.eventType)).toEqual([ + 'step_created', + 'step_started', + ]); + expect(events[0].eventId < events[1].eventId).toBe(true); + // The input belongs to the synthetic step_created, not the started row. + expect( + (events[0] as { eventData: { input: unknown } }).eventData.input + ).toEqual(new Uint8Array([7])); + expect( + (events[1] as { eventData?: { input?: unknown } }).eventData?.input + ).toBeUndefined(); + }); + + it('treats a lazy step_started for an existing step as a lost create race', async () => { + const lazy = { + eventType: 'step_started', + specVersion: SPEC, + correlationId: 'step_1', + eventData: { stepName: 'step//./w//charge', input: new Uint8Array() }, + } as const; + await store.events.create(RUN, lazy); + // Exactly-once ownership: the loser must be told to skip, not allowed to + // re-run the body. + await expect(store.events.create(RUN, lazy)).rejects.toBeInstanceOf( + EntityConflictError + ); + }); + + it('honours retryAfter', async () => { + await store.events.create(RUN, { + eventType: 'step_created', + specVersion: SPEC, + correlationId: 'step_1', + eventData: { stepName: 'step//./w//charge', input: new Uint8Array() }, + }); + await store.events.create(RUN, { + eventType: 'step_started', + specVersion: SPEC, + correlationId: 'step_1', + }); + await store.events.create(RUN, { + eventType: 'step_retrying', + specVersion: SPEC, + correlationId: 'step_1', + eventData: { + error: new Uint8Array(), + retryAfter: new Date(1_704_067_200_000 + 10_000), + }, + }); + await expect( + store.events.create(RUN, { + eventType: 'step_started', + specVersion: SPEC, + correlationId: 'step_1', + }) + ).rejects.toThrow(/retryAfter/); + + tick(10_000); + await expect( + store.events.create(RUN, { + eventType: 'step_started', + specVersion: SPEC, + correlationId: 'step_1', + }) + ).resolves.toBeTruthy(); + }); + }); + + describe('hooks', () => { + beforeEach(() => createRun(store, RUN)); + + const hookCreated = (hookId: string, token: string) => + ({ + eventType: 'hook_created', + specVersion: SPEC, + correlationId: hookId, + eventData: { token }, + }) as const; + + it('grants a token to one live hook at a time and journals conflicts', async () => { + await store.events.create(RUN, hookCreated('hook_1', 'approval:1')); + const conflict = await store.events.create( + RUN, + hookCreated('hook_2', 'approval:1') + ); + // A conflict is data the workflow must observe, not an exception the + // caller has to handle. + expect(conflict.event?.eventType).toBe('hook_conflict'); + expect(conflict.hook).toBeUndefined(); + expect(store.hookByToken('approval:1')?.hookId).toBe('hook_1'); + }); + + it('releases the token on dispose and refuses later resumes', async () => { + await store.events.create(RUN, hookCreated('hook_1', 'approval:1')); + await store.events.create(RUN, { + eventType: 'hook_disposed', + specVersion: SPEC, + correlationId: 'hook_1', + }); + expect(store.hookByToken('approval:1')).toBeUndefined(); + await expect( + store.events.create(RUN, { + eventType: 'hook_received', + specVersion: SPEC, + correlationId: 'hook_1', + eventData: { payload: new Uint8Array() }, + }) + ).rejects.toBeInstanceOf(HookNotFoundError); + + // The token is free again for a new hook. + await expect( + store.events.create(RUN, hookCreated('hook_2', 'approval:1')) + ).resolves.toMatchObject({ hook: { hookId: 'hook_2' } }); + }); + + it('carries a resume context so a resume needs no run read', async () => { + const result = await store.events.create( + RUN, + hookCreated('hook_1', 'approval:1') + ); + expect(result.hook?.resumeContext).toMatchObject({ + deploymentId: 'dpl_sim', + workflowName: 'workflow//./w//demo', + }); + }); + }); + + describe('terminal runs', () => { + beforeEach(() => createRun(store, RUN)); + + it('rejects new entities but accepts the terminal write of a running step', async () => { + await store.events.create(RUN, { + eventType: 'step_started', + specVersion: SPEC, + correlationId: 'step_1', + eventData: { stepName: 'step//./w//charge', input: new Uint8Array() }, + }); + await store.events.create(RUN, { + eventType: 'run_cancelled', + specVersion: SPEC, + }); + + await expect( + store.events.create(RUN, { + eventType: 'hook_created', + specVersion: SPEC, + correlationId: 'hook_1', + eventData: { token: 'approval:1' }, + }) + ).rejects.toBeInstanceOf(EntityConflictError); + + // The in-flight step still gets to report back β€” that write is how an + // inline step body closes itself out. + await expect( + store.events.create(RUN, { + eventType: 'step_completed', + specVersion: SPEC, + correlationId: 'step_1', + eventData: { result: new Uint8Array() }, + }) + ).resolves.toBeTruthy(); + }); + + it('maps run_started on a terminal run to RunExpiredError', async () => { + await store.events.create(RUN, { + eventType: 'run_completed', + specVersion: SPEC, + eventData: {}, + }); + await expect( + store.events.create(RUN, { + eventType: 'run_started', + specVersion: SPEC, + }) + ).rejects.toBeInstanceOf(RunExpiredError); + }); + + it('is idempotent for a repeated cancel', async () => { + await store.events.create(RUN, { + eventType: 'run_cancelled', + specVersion: SPEC, + }); + await expect( + store.events.create(RUN, { + eventType: 'run_cancelled', + specVersion: SPEC, + }) + ).resolves.toMatchObject({ run: { status: 'cancelled' } }); + }); + }); + + describe('pagination', () => { + it('pages events ascending with a resumable cursor', async () => { + await createRun(store, RUN); + for (let i = 0; i < 5; i++) { + tick(1); + await store.events.create(RUN, { + eventType: 'step_created', + specVersion: SPEC, + correlationId: `step_${i}`, + eventData: { stepName: 'step//./w//s', input: new Uint8Array() }, + }); + } + const first = await store.events.list({ + runId: RUN, + pagination: { limit: 3, sortOrder: 'asc' }, + }); + expect(first.data).toHaveLength(3); + expect(first.hasMore).toBe(true); + + const second = await store.events.list({ + runId: RUN, + pagination: { + limit: 10, + sortOrder: 'asc', + cursor: first.cursor ?? undefined, + }, + }); + expect(second.hasMore).toBe(false); + expect([...first.data, ...second.data].map((e) => e.eventId)).toEqual( + store.allEvents(RUN).map((e) => e.eventId) + ); + }); + }); + + describe('precondition guard', () => { + it('fences a replay write behind an out-of-band one, only when enabled', async () => { + const guarded = setup({ preconditionGuard: true }); + await createRun(guarded.store, RUN); + await guarded.store.events.create(RUN, { + eventType: 'hook_created', + specVersion: SPEC, + correlationId: 'hook_1', + eventData: { token: 'approval:1' }, + }); + + guarded.tick(10); + const snapshot = guarded.nowMs(); + guarded.tick(10); + // An out-of-band resume: no stateUpdatedAt, so it advances the marker. + await guarded.store.events.create(RUN, { + eventType: 'hook_received', + specVersion: SPEC, + correlationId: 'hook_1', + eventData: { payload: new Uint8Array() }, + }); + + await expect( + guarded.store.events.create( + RUN, + { + eventType: 'step_created', + specVersion: SPEC, + correlationId: 'step_1', + eventData: { stepName: 'step//./w//s', input: new Uint8Array() }, + }, + { stateUpdatedAt: snapshot } + ) + ).rejects.toThrow(/out of band/); + + // An up-to-date snapshot passes β€” an equal timestamp must not livelock. + await expect( + guarded.store.events.create( + RUN, + { + eventType: 'step_created', + specVersion: SPEC, + correlationId: 'step_1', + eventData: { stepName: 'step//./w//s', input: new Uint8Array() }, + }, + { stateUpdatedAt: guarded.nowMs() } + ) + ).resolves.toBeTruthy(); + }); + }); + + describe('append-only log', () => { + const hook = { + eventType: 'hook_created', + specVersion: SPEC, + correlationId: 'hook_1', + eventData: { token: 'approval:1' }, + } as const; + const step = { + eventType: 'step_created', + specVersion: SPEC, + correlationId: 'step_1', + eventData: { stepName: 'step//./w//s', input: new Uint8Array() }, + } as const; + + it('is off by default: a held write lands behind one committed sooner', async () => { + await createRun(store, RUN); + // The handler boundary takes a position; the write is then held. + const minted = store.mintEvent(); + tick(10); + const overtook = await store.events.create(RUN, hook); + const held = await store.events.create(RUN, step, heldAt(minted)); + + expect(held.event?.eventId).toBe(minted.eventId); + // The log gained a row in the past: the later commit sorts first, so a + // reader that already saw `overtook` has been passed by something older. + const ids = store.allEvents(RUN).map((e) => e.eventId); + expect(ids.indexOf(minted.eventId)).toBeLessThan( + ids.indexOf(overtook.event?.eventId ?? '') + ); + }); + + it('re-mints a write that was overtaken while it was held', async () => { + const world = setup({ appendOnlyLog: true }); + await createRun(world.store, RUN); + const minted = world.store.mintEvent(); + world.tick(10); + const overtook = await world.store.events.create(RUN, hook); + const held = await world.store.events.create(RUN, step, heldAt(minted)); + + // The reserved position is abandoned. Nothing is ever inserted behind a + // row a reader could already have seen, so log order is commit order. + const heldId = held.event?.eventId ?? ''; + expect(heldId).not.toBe(minted.eventId); + expect(heldId > (overtook.event?.eventId ?? '')).toBe(true); + const ids = world.store.allEvents(RUN).map((e) => e.eventId); + expect(ids).toEqual([...ids].sort()); + expect(ids.at(-1)).toBe(heldId); + }); + + it('leaves an uncontended write at the position it minted', async () => { + const world = setup({ appendOnlyLog: true }); + await createRun(world.store, RUN); + const minted = world.store.mintEvent(); + world.tick(10); + const uncontended = await world.store.events.create( + RUN, + step, + heldAt(minted) + ); + + // Still the newest position when it arrived, so it keeps both halves of + // the mint β€” including a `createdAt` from before the tick. A scenario + // that never holds a write mid-flight logs the same bytes either way. + expect(uncontended.event?.eventId).toBe(minted.eventId); + expect(uncontended.event?.createdAt).toEqual(minted.createdAt); + }); + + it('punches a hole in a withheld read by default', async () => { + const seen: StaleRead[] = []; + const world = setup({ onStaleRead: (read) => seen.push(read) }); + await createRun(world.store, RUN); + world.store.withholdNextEvent(1); + const withheld = await world.store.events.create(RUN, hook); + world.tick(1); + const after = await world.store.events.create(RUN, step); + + const page = await world.store.events.list({ + runId: RUN, + pagination: { limit: 50, sortOrder: 'asc' }, + }); + const ids = page.data.map((e) => e.eventId); + // The withheld event vanishes and its successor stays: the reader holds + // proof that something newer exists, which is what no watermark can see. + expect(ids).not.toContain(withheld.event?.eventId); + expect(ids).toContain(after.event?.eventId); + expect(seen).toEqual([ + { eventId: withheld.event?.eventId, hidden: 1, truncated: false }, + ]); + }); + + it('truncates a withheld read rather than punching a hole', async () => { + const seen: StaleRead[] = []; + const world = setup({ + appendOnlyLog: true, + onStaleRead: (read) => seen.push(read), + }); + await createRun(world.store, RUN); + world.store.withholdNextEvent(1); + const withheld = await world.store.events.create(RUN, hook); + world.tick(1); + const after = await world.store.events.create(RUN, step); + + const page = await world.store.events.list({ + runId: RUN, + pagination: { limit: 50, sortOrder: 'asc' }, + }); + const ids = page.data.map((e) => e.eventId); + // Cut short at the withheld event: a prefix of the real log. Still short + // β€” both events are missing, `after` included β€” but not self-contradictory. + expect(ids).not.toContain(withheld.event?.eventId); + expect(ids).not.toContain(after.event?.eventId); + expect(ids).toEqual( + world.store + .allEvents(RUN) + .map((e) => e.eventId) + .filter((id) => id < (withheld.event?.eventId ?? '')) + ); + expect(seen).toEqual([ + { eventId: withheld.event?.eventId, hidden: 2, truncated: true }, + ]); + }); + + it('serves the real log once the withhold window closes', async () => { + const world = setup({ appendOnlyLog: true }); + await createRun(world.store, RUN); + world.store.withholdNextEvent(1); + await world.store.events.create(RUN, hook); + + const read = () => + world.store.events + .list({ runId: RUN, pagination: { limit: 50, sortOrder: 'asc' } }) + .then((p) => p.data.length); + const short = await read(); + expect(await read()).toBe(short + 1); + }); + }); + + describe('seeding a log', () => { + // `seedFromLog` folds a committed log back into entity rows, and it is the + // path a replay check cold-starts from. If that fold disagrees with the one + // the write path runs, the replay diverges from the run it is checking for + // reasons that have nothing to do with the runtime under test β€” so the + // property worth pinning is that the two agree. + // + // Timestamps line up for free: an event's `createdAt` is minted inside the + // same `create` call that commits it, so the seeded row and the written row + // read the same clock even when the test ticks between writes. + const seededFrom = (source: SimStore): SimStore => { + const { store: fresh } = setup(); + fresh.seedFromLog(source.allEvents()); + return fresh; + }; + + const expectSameEntities = (source: SimStore, fresh: SimStore) => { + expect(fresh.allRuns()).toEqual(source.allRuns()); + expect(fresh.allSteps()).toEqual(source.allSteps()); + expect(fresh.allHooks()).toEqual(source.allHooks()); + expect(fresh.allWaits()).toEqual(source.allWaits()); + }; + + it('rebuilds the run and its steps', async () => { + await createRun(store, RUN); + await store.events.create(RUN, { + eventType: 'attr_set', + specVersion: SPEC, + eventData: { + changes: [ + { key: 'kept', value: 'yes' }, + { key: 'dropped', value: null }, + ], + }, + }); + await store.events.create(RUN, { + eventType: 'step_created', + specVersion: SPEC, + correlationId: 'step_1', + eventData: { + stepName: 'step//./w//charge', + input: new Uint8Array([1]), + }, + }); + await store.events.create(RUN, { + eventType: 'step_started', + specVersion: SPEC, + correlationId: 'step_1', + }); + await store.events.create(RUN, { + eventType: 'step_completed', + specVersion: SPEC, + correlationId: 'step_1', + eventData: { result: new Uint8Array([2]) }, + }); + // A lazy start, so the seeded fold has to pick up the synthetic + // `step_created` the write path wrote alongside it. + await store.events.create(RUN, { + eventType: 'step_started', + specVersion: SPEC, + correlationId: 'step_2', + eventData: { stepName: 'step//./w//ship', input: new Uint8Array([3]) }, + }); + await store.events.create(RUN, { + eventType: 'step_retrying', + specVersion: SPEC, + correlationId: 'step_2', + eventData: { error: new Uint8Array([4]) }, + }); + tick(1); + await store.events.create(RUN, { + eventType: 'step_started', + specVersion: SPEC, + correlationId: 'step_2', + }); + + const fresh = seededFrom(store); + expectSameEntities(store, fresh); + // The rows are not trivially empty on either side. + expect(fresh.allSteps(RUN).map((s) => s.status)).toEqual([ + 'completed', + 'running', + ]); + expect(fresh.allSteps(RUN)[1].attempt).toBe(2); + expect(fresh.allRuns()[0].attributes).toEqual({ kept: 'yes' }); + }); + + it('rebuilds hooks and waits, including who owns a token', async () => { + await createRun(store, RUN); + await store.events.create(RUN, { + eventType: 'hook_created', + specVersion: SPEC, + correlationId: 'hook_1', + eventData: { token: 'approval:1' }, + }); + await store.events.create(RUN, { + eventType: 'hook_received', + specVersion: SPEC, + correlationId: 'hook_1', + eventData: { payload: new Uint8Array([5]) }, + }); + await store.events.create(RUN, { + eventType: 'hook_created', + specVersion: SPEC, + correlationId: 'hook_2', + eventData: { token: 'approval:2' }, + }); + await store.events.create(RUN, { + eventType: 'hook_disposed', + specVersion: SPEC, + correlationId: 'hook_2', + }); + await store.events.create(RUN, { + eventType: 'wait_created', + specVersion: SPEC, + correlationId: 'wait_1', + eventData: { resumeAt: new Date(1_704_067_260_000) }, + }); + await store.events.create(RUN, { + eventType: 'wait_completed', + specVersion: SPEC, + correlationId: 'wait_1', + }); + + const fresh = seededFrom(store); + expectSameEntities(store, fresh); + // A disposed hook releases its token on both paths; a live one keeps it. + expect(fresh.hookByToken('approval:1')).toEqual( + store.hookByToken('approval:1') + ); + expect(fresh.hookByToken('approval:2')).toBeUndefined(); + expect(fresh.allWaits(RUN)[0].status).toBe('completed'); + }); + + it('releases the hooks and waits of a terminated run', async () => { + await createRun(store, RUN); + await store.events.create(RUN, { + eventType: 'hook_created', + specVersion: SPEC, + correlationId: 'hook_1', + eventData: { token: 'approval:1' }, + }); + await store.events.create(RUN, { + eventType: 'wait_created', + specVersion: SPEC, + correlationId: 'wait_1', + eventData: { resumeAt: new Date(1_704_067_260_000) }, + }); + await store.events.create(RUN, { + eventType: 'run_completed', + specVersion: SPEC, + eventData: { output: new Uint8Array([6]) }, + }); + + const fresh = seededFrom(store); + expectSameEntities(store, fresh); + expect(fresh.allHooks()).toEqual([]); + expect(fresh.allWaits()).toEqual([]); + expect(fresh.allRuns()[0].status).toBe('completed'); + }); + }); +}); diff --git a/packages/world-sim/src/store.ts b/packages/world-sim/src/store.ts new file mode 100644 index 0000000000..c2d007279d --- /dev/null +++ b/packages/world-sim/src/store.ts @@ -0,0 +1,1449 @@ +/** + * In-memory, single-threaded event store. + * + * This is a reference implementation of the World storage contract: the same + * event β†’ entity state machine `@workflow/world-local` implements on the + * filesystem, minus every mechanism that exists purely to make that state + * machine safe against concurrent processes (exclusive-create claim files, + * per-entity file locks, staged/promoted hook events, canonical event-id + * pinning after a crash). A scenario runs exactly one delivery at a time in + * one process, so those races cannot occur here and their absence is what + * keeps this file small enough to audit. + * + * What is deliberately *kept* is every validation that rejects an event: + * terminal-run guards, step lifecycle ordering, hook token uniqueness, wait + * duplication. Those rejections are the observable contract the runtime is + * written against, so a simulation that relaxed them would agree with the + * runtime about nothing interesting. + */ + +import { + EntityConflictError, + HookNotFoundError, + PreconditionFailedError, + RunExpiredError, + TooEarlyError, + WorkflowRunNotFoundError, + WorkflowWorldError, +} from '@workflow/errors'; +import { + type AnyEventRequest, + type CreateEventParams, + type Event, + type EventResult, + type Hook, + type HookResumeContext, + isChildEntityCreationEvent, + isHookEventRequiringExistence, + isStepEventType, + isTerminalRunEventType, + isTerminalStepEventType, + isTerminalStepStatus, + isTerminalWorkflowRunStatus, + type PaginatedResponse, + type PaginationOptions, + type ResolveData, + SPEC_VERSION_CURRENT, + type Step, + type Storage, + stripEventDataRefs, + type Wait, + type WorkflowRun, +} from '@workflow/world'; +import { type IdFactory, ulidTimeOf } from './ids.js'; + +/** Per-run event ceiling reported on run responses, mirroring the other worlds. */ +const MAX_EVENTS_PER_RUN = 25_000; + +const DEFAULT_PAGE_LIMIT = 20; + +/** + * How many of a run's most recent event ids the count guard keeps. + * + * Mirrors workflow-server's `RUN_EVENT_INDEX_WINDOW`. The window is what makes + * the guard one-sided: a hole deeper than this cannot be proven, so the + * comparison reports `indeterminate` and the write is allowed through. + */ +const RUN_EVENT_INDEX_WINDOW = 16; + +/** + * A log position, minted before the write that will occupy it commits. + * + * The event id *is* the log's sort key, and the sim mints it the way + * workflow-server does: in the request handler (`EventId.make()`), not in + * storage β€” DynamoDB does not generate ids. Everything downstream follows from + * that one fact. A write that is minted and then takes a while to commit keeps + * the earlier position it was given, so the log can gain an event *behind* a + * position a reader has already seen. That is the hole no high-water mark can + * detect, and reproducing it is the reason minting is separable from appending. + * + * `createdAt` is the mint instant, not the commit instant: in production it is + * decoded back out of the ULID, so the two can never disagree. Entity rows + * (step/run/hook timestamps) still use the commit instant, because those are + * written by the transaction rather than derived from the id. + */ +export interface MintedEvent { + eventId: string; + createdAt: Date; +} + +/** + * Sim-internal `events.create` params, supplied by the world facade rather than + * by the runtime under test. + */ +interface SimCreateParams { + /** + * The position minted for this write at the handler boundary. Absent when the + * store is driven directly (unit tests), in which case `create` mints on + * entry β€” the same instant, just without a hold point in between. + */ + minted?: MintedEvent; + /** + * How many events the caller had loaded when it decided to make this write. + * Mirrors workflow-server's `stateEventCount`, and since #3145 the runtime + * sends it; see `SimStoreOptions.countGuard`. + */ + stateEventCount?: number; +} + +/** Per run: the tail of the log, for the count guard. See `countRecordedAtOrBelow`. */ +interface RunEventIndex { + recentEventIds: string[]; + total: number; +} + +/** + * How many events the log holds at or below `stateUpdatedAt`, or `null` when + * the retained window cannot prove it. + * + * Ported from workflow-server's `countRecordedAtOrBelow`, including its + * exactness argument: pruning always drops the oldest id, so `total - above` is + * exact whenever the window still reaches back past the snapshot. The one case + * it refuses to evaluate is a pruned window whose every retained id is above + * the snapshot β€” the dropped ids may have been above it too. + */ +function countRecordedAtOrBelow( + index: RunEventIndex, + stateUpdatedAt: number +): number | null { + const above = index.recentEventIds.filter( + (id) => ulidTimeOf(id) > stateUpdatedAt + ).length; + const pruned = index.total > index.recentEventIds.length; + if (pruned && above === index.recentEventIds.length) return null; + return index.total - above; +} + +export interface SimStoreOptions { + now(): number; + ids: IdFactory; + /** + * Enforce the optimistic-concurrency precondition guard described in + * `WorldCapabilities.preconditionGuard`: reject a replay-context write whose + * `stateUpdatedAt` snapshot predates the newest externally-originated event. + * + * Off by default. Turning it on is the point of a simulation β€” it lets a + * scenario check that the runtime recovers from a 412 fence β€” but it also + * changes which runtime fast paths engage, so it is never implicit. + */ + preconditionGuard?: boolean; + /** + * Also enforce workflow-server's *count* guard: reject a write whose caller + * loaded fewer events at or below its own watermark than the log actually + * holds there. + * + * This is the half of the fence the watermark cannot express. A high-water + * mark answers "is there anything newer than my snapshot?", which sees a log + * truncated at the end; the count answers "is anything missing *behind* my + * snapshot?", which is the hole two concurrent writers actually produce. + * + * Requires `preconditionGuard` (it reuses `stateUpdatedAt` as the watermark to + * count against) and a client that sends `stateEventCount` β€” which + * `@workflow/core` has done on every replay-context create since #3145. The + * sim uses that value when it is there and reconstructs one for the writes + * core does not count; see `SimWorldOptions.countGuard`. + */ + countGuard?: boolean; + /** + * Make the log append-only in the strong sense: an event takes its position + * when it *commits*, not when its handler minted one. + * + * Production does the opposite, and for a reason β€” DynamoDB does not + * generate ids, so workflow-server mints the event id at the handler + * boundary and that id *is* the log's sort key. A write held between its + * mint and its commit therefore lands *behind* events that were minted later + * and committed sooner, and the log gains a row in the past. Every read that + * happened in between saw a log the log itself went on to contradict. That + * one fact is what the six red scenarios in the book are about. + * + * With this on, a write that was overtaken while it was held gives up its + * reserved position and re-mints at the tail. Two consequences follow, and + * they are the whole point: + * + * - Log order is commit order. Nothing is ever inserted behind a row a + * reader has already seen, so no two reads can disagree about the past. + * - Every read is a prefix of the final log. A read can be *short* β€” it may + * miss a write that has not committed yet, or one a lagging replica has + * not caught up to (see `withholdNextEvent`) β€” but never self-inconsistent. + * Staleness collapses into lag, and lag is what the optimistic-concurrency + * fence can see; a hole is what it cannot. + * + * What it costs is the property the boundary mint was buying: a write no + * longer knows where it will land until it lands. Off by default, because + * the simulation's job is to model the world that exists. Turning it on + * answers the other question β€” which of these failures survive if it didn't? + * + * Uncontended writes are untouched. A mint that is still the newest position + * when it commits keeps its id, so a scenario that never holds a write + * mid-flight produces a byte-identical log either way. + */ + appendOnlyLog?: boolean; + /** Invoked after every successful append, before the create call returns. */ + onEvent?(event: Event): void; + /** Invoked when a read was served an incomplete log. */ + onStaleRead?(read: StaleRead): void; +} + +/** One event-log read that did not see everything the log already held. */ +export interface StaleRead { + /** The oldest event the read did not see. */ + eventId: string; + /** How many committed events the read did not see, that one included. */ + hidden: number; + /** + * The read was cut short at `eventId` rather than served around it: a + * replica that is behind, not one that is wrong. Only `appendOnlyLog` + * produces this shape β€” see there. + */ + truncated: boolean; +} + +export interface SimStore extends Storage { + /** + * Load a previously committed log into an empty store, verbatim β€” same + * event ids, same timestamps β€” and fold the entity state back out of it. + * + * This is the "cold start" primitive: it reconstructs the durable state a + * fresh process would find, without re-validating writes that were already + * accepted once. Seeded events are deliberately not reported to `onEvent`, + * so a trace of the seeded world shows only what the replay newly derives. + */ + seedFromLog(log: readonly Event[]): void; + /** + * Mint the next log position, without writing anything. + * + * The world facade calls this at the handler boundary β€” before any hold can + * fire β€” so a write held mid-flight already owns the position it will + * eventually occupy. See {@link MintedEvent}. + */ + mintEvent(): MintedEvent; + /** + * Hide the *next* event appended from the following `reads` event-log reads. + * + * This models one concurrent writer precisely. Under real concurrency two + * writers take positions 7 and 8, and a reader can observe 8 while 7 is still + * in flight β€” a *hole*, not a truncated tail. Withholding a suffix instead + * would hide the reader's own write too, which is a different (and less + * interesting) fault. + * + * It is the second of the three preconditions for a corrupted event log β€” a + * write derived from an incomplete event load. A strictly serial scheduler + * cannot reach it by accident, so a scenario has to ask for it. + * + * Note which world this models. Hiding an event that is already *committed* + * is a stale read, and workflow-server has eliminated those: it pays 2Γ— the + * RCU for strongly-consistent reads on every page, so every event committed + * before a read started is visible to it. This primitive therefore models an + * eventually-consistent backend (or the older split-query world-vercel read + * path), and it is the *weaker* fault of the two. The stronger one needs no + * withholding at all: hold a write between its mint and its commit and the + * reader genuinely cannot see it, because it is not there yet β€” while its + * position, already assigned, sits behind whatever the reader did see. Prefer + * a hold when the scenario's point is production behaviour. + * + * Under `appendOnlyLog` the hole is not expressible, so this degrades to the + * honest version of the same lag: the read stops *at* the withheld event + * instead of stepping over it. The reader still misses the write; what it no + * longer does is miss it while holding proof that something newer exists. + */ + withholdNextEvent(reads?: number): void; + /** Every event ever appended, in log order. */ + allEvents(runId?: string): Event[]; + /** + * The same events in the order they were *committed*, which is the order this + * array was appended to. Differs from `allEvents` exactly when a write was + * minted before another and committed after it β€” so the two together are what + * `log.monotonic-order` compares. + */ + allEventsInCommitOrder(runId?: string): Event[]; + allRuns(): WorkflowRun[]; + allSteps(runId?: string): Step[]; + allHooks(runId?: string): Hook[]; + allWaits(runId?: string): Wait[]; + hookByToken(token: string): Hook | undefined; +} + +/** Brand check that survives a swapped global constructor (see `clock.ts`). */ +function isDate(value: unknown): value is Date { + return Object.prototype.toString.call(value) === '[object Date]'; +} + +function clone(value: T): T { + if (Array.isArray(value)) return value.map(clone) as unknown as T; + // Deliberately not `instanceof`: a Date minted under one virtual clock must + // still read as a Date under the next one. Getting this wrong turns a Date + // into `{}` (it has no own enumerable properties) far from the actual bug. + if (isDate(value)) return new Date((value as Date).getTime()) as unknown as T; + if (value instanceof Uint8Array) return value as unknown as T; + if (value && typeof value === 'object') { + const out: Record = {}; + for (const [k, v] of Object.entries(value)) out[k] = clone(v); + return out as T; + } + return value; +} + +function encodeCursor(createdAt: Date, id: string): string { + return `${createdAt.toISOString()}|${id}`; +} + +function decodeCursor( + cursor: string | undefined +): { timeMs: number; id: string | null } | null { + if (!cursor) return null; + const [time, id] = cursor.split('|'); + return { timeMs: new Date(time).getTime(), id: id || null }; +} + +/** + * Shared pagination over an in-memory collection, matching world-local's + * `(createdAt, id)` ordering and `"|"` cursor format exactly. The + * runtime pages through event logs with these semantics, so a divergence here + * would show up as phantom replay divergence rather than as a store bug. + */ +function paginate( + items: readonly T[], + opts: { + pagination?: PaginationOptions; + defaultSortOrder?: 'asc' | 'desc'; + getCreatedAt(item: T): Date; + getId(item: T): string; + } +): PaginatedResponse { + const sortOrder = + opts.pagination?.sortOrder ?? opts.defaultSortOrder ?? 'desc'; + const limit = opts.pagination?.limit ?? DEFAULT_PAGE_LIMIT; + const cursor = decodeCursor(opts.pagination?.cursor); + + const sorted = [...items].sort((a, b) => { + const at = opts.getCreatedAt(a).getTime(); + const bt = opts.getCreatedAt(b).getTime(); + if (at !== bt) return sortOrder === 'asc' ? at - bt : bt - at; + const ai = opts.getId(a); + const bi = opts.getId(b); + return sortOrder === 'asc' ? ai.localeCompare(bi) : bi.localeCompare(ai); + }); + + const afterCursor = cursor + ? sorted.filter((item) => { + const t = opts.getCreatedAt(item).getTime(); + if (sortOrder === 'asc') { + if (t < cursor.timeMs) return false; + if (t === cursor.timeMs && cursor.id) { + return opts.getId(item).localeCompare(cursor.id) > 0; + } + return t > cursor.timeMs; + } + if (t > cursor.timeMs) return false; + if (t === cursor.timeMs && cursor.id) { + return opts.getId(item).localeCompare(cursor.id) < 0; + } + return t < cursor.timeMs; + }) + : sorted; + + const hasMore = afterCursor.length > limit; + const page = hasMore ? afterCursor.slice(0, limit) : afterCursor; + const last = page[page.length - 1]; + return { + data: page.map(clone), + cursor: last + ? encodeCursor(opts.getCreatedAt(last), opts.getId(last)) + : null, + hasMore, + }; +} + +/** What one event changed. Empty when the event owns no entity. */ +interface AppliedEntities { + run?: WorkflowRun; + step?: Step; + hook?: Hook; + wait?: Wait; +} + +export function createSimStore(options: SimStoreOptions): SimStore { + const { ids, now: nowMs } = options; + const appendOnlyLog = options.appendOnlyLog === true; + + const events: Event[] = []; + const runs = new Map(); + /** Keyed `${runId}:${stepId}`. */ + const steps = new Map(); + const hooks = new Map(); + /** Live token β†’ hookId. A disposed or run-terminated hook releases its token. */ + const tokenOwners = new Map(); + /** Keyed `${runId}:${correlationId}`. */ + const waits = new Map(); + /** hookIds that have been explicitly disposed; disposal is permanent. */ + const disposedHooks = new Set(); + /** + * Per run: ULID time of the newest externally-originated event. Only read + * when `preconditionGuard` is on. See `CreateEventParams.stateUpdatedAt`. + */ + const externalWriteMarker = new Map(); + /** + * Per run: the tail of the log, for the count guard. Records *every* event, + * replay-origin included β€” the corruption it guards against is one replay + * racing another, which the out-of-band marker cannot see by construction. + */ + const runEventIndex = new Map(); + + /** Reads to withhold the next appended event from, once it is appended. */ + let armedWithhold: number | undefined; + /** The withheld event and how many more reads must not see it. */ + let withheld: { eventId: string; remaining: number } | undefined; + + /** + * Serve a read, minus any event currently being withheld. Reads outside a + * withhold window get the real log. + * + * Both modes hide the same event and differ only in what they do with the + * ones behind it. The default punches a hole β€” the withheld event vanishes + * and its successors stay β€” which is what an eventually-consistent replica + * does and what no watermark can detect. Under `appendOnlyLog` the read is + * cut short there instead, leaving a prefix: still short, but no longer + * carrying evidence that contradicts itself. + */ + function applyWithhold(source: readonly Event[]): readonly Event[] { + if (!withheld || withheld.remaining <= 0) return source; + const { eventId } = withheld; + withheld.remaining--; + if (withheld.remaining <= 0) withheld = undefined; + const visible = appendOnlyLog + ? source.filter((e) => e.eventId < eventId) + : source.filter((e) => e.eventId !== eventId); + const hidden = source.length - visible.length; + if (hidden > 0) { + options.onStaleRead?.({ eventId, hidden, truncated: appendOnlyLog }); + } + return visible; + } + + const stepKey = (runId: string, stepId: string) => `${runId}:${stepId}`; + const waitKey = (runId: string, correlationId: string) => + `${runId}:${correlationId}`; + + function mintEvent(): MintedEvent { + return { eventId: ids.eventId(), createdAt: new Date(nowMs()) }; + } + + function recordInIndex(event: Event): void { + const index = runEventIndex.get(event.runId) ?? { + recentEventIds: [], + total: 0, + }; + index.recentEventIds.push(event.eventId); + // Keep the window in mint order, so "oldest id" and "oldest event" stay the + // same thing β€” `countRecordedAtOrBelow`'s exactness argument depends on it. + index.recentEventIds.sort((a, b) => a.localeCompare(b)); + if (index.recentEventIds.length > RUN_EVENT_INDEX_WINDOW) { + index.recentEventIds.shift(); + } + index.total++; + runEventIndex.set(event.runId, index); + } + + /** + * The position an event actually commits at. + * + * Only `appendOnlyLog` can move one. A write that is still the newest + * position when it arrives keeps the id it was already handed out under, so + * uncontended history is unchanged; one that was overtaken while it was held + * re-mints and takes the tail. + * + * Compared as plain strings: the ids are fixed-width ULIDs whose lexical + * order *is* `(createdAt, eventId)` order, because the time field is the + * `createdAt` millisecond and the suffix is a monotonic counter. See + * `createIdFactory`. + */ + function positionAtCommit(event: Event): Event { + if (!appendOnlyLog) return event; + const tail = events[events.length - 1]; + if (!tail || event.eventId > tail.eventId) return event; + return { ...event, ...mintEvent() }; + } + + function append(incoming: Event): Event { + const event = positionAtCommit(incoming); + events.push(event); + recordInIndex(event); + if (armedWithhold !== undefined) { + withheld = { eventId: event.eventId, remaining: armedWithhold }; + armedWithhold = undefined; + } + options.onEvent?.(event); + return event; + } + + function requireRun(runId: string): WorkflowRun { + const run = runs.get(runId); + if (!run) throw new WorkflowRunNotFoundError(runId); + return run; + } + + function resumeContextFor(run: WorkflowRun): HookResumeContext { + const ctx = run.executionContext ?? {}; + return { + deploymentId: run.deploymentId, + workflowName: run.workflowName, + runSpecVersion: run.specVersion, + ...(typeof ctx.workflowCoreVersion === 'string' + ? { workflowCoreVersion: ctx.workflowCoreVersion } + : {}), + ...(ctx.traceCarrier && typeof ctx.traceCarrier === 'object' + ? { + traceCarrier: ctx.traceCarrier as HookResumeContext['traceCarrier'], + } + : {}), + ...(run.encryptionPublicKey + ? { encryptionPublicKey: run.encryptionPublicKey } + : {}), + }; + } + + /** + * Release the hooks and waits a terminated run owned. Mirrors the other + * worlds: once a run is terminal its hooks can never be resumed, so their + * tokens become available again. + */ + function releaseRunResources(runId: string) { + for (const [hookId, hook] of hooks) { + if (hook.runId !== runId) continue; + if (tokenOwners.get(hook.token) === hookId) + tokenOwners.delete(hook.token); + hooks.delete(hookId); + } + for (const [key, wait] of waits) { + if (wait.runId === runId) waits.delete(key); + } + } + + function eventsForRun(runId: string): Event[] { + return events.filter((e) => e.runId === runId); + } + + /** + * Apply one event to the entity rows, and report what it touched. + * + * The single copy of the event β†’ entity state machine. Both paths into the + * store end here: `create` runs its validation and then calls this, and + * `seedFromLog` calls it with no validation at all β€” those events were + * accepted once already, and re-litigating them would reject legitimate + * history (a `step_completed` recorded after the run was cancelled, say). + * + * So the applier is *total*: an event whose subject is missing is a no-op + * rather than an error, and refusing anything is the caller's job. Holding + * both paths to one fold is what keeps a replay from diverging from the run + * it is checking for a reason that is not the runtime's fault. + * + * `at` is the entity timestamp: commit time on the write path; the event's + * own position time when seeding, where there is no live clock to read. + */ + function applyEvent(event: Event, at: Date): AppliedEntities { + const runId = event.runId; + const data = (event as { eventData?: Record }).eventData; + const correlationId = event.correlationId; + + switch (event.eventType) { + case 'run_created': { + const run = { + runId, + deploymentId: data?.deploymentId as string, + workflowName: data?.workflowName as string, + status: 'pending', + specVersion: event.specVersion, + executionContext: data?.executionContext as Record, + input: data?.input as Uint8Array, + attributes: (data?.attributes as Record) ?? {}, + encryptionPublicKey: data?.encryptionPublicKey as string | undefined, + createdAt: at, + updatedAt: at, + } as WorkflowRun; + runs.set(runId, run); + return { run }; + } + + case 'run_started': { + const existing = runs.get(runId); + if (!existing) return {}; + // The clears are for the write path, where a restart is a real + // transition. On a seeded log they are already undefined: a + // `run_started` never follows a terminal event in a log the write path + // accepted. + const run = { + ...existing, + status: 'running', + output: undefined, + error: undefined, + completedAt: undefined, + startedAt: existing.startedAt ?? at, + updatedAt: at, + } as WorkflowRun; + runs.set(runId, run); + return { run }; + } + + case 'run_completed': + case 'run_failed': + case 'run_cancelled': { + const existing = runs.get(runId); + if (!existing) return {}; + const run = { + ...existing, + status: + event.eventType === 'run_completed' + ? 'completed' + : event.eventType === 'run_failed' + ? 'failed' + : 'cancelled', + output: data?.output as Uint8Array | undefined, + error: data?.error as Uint8Array | undefined, + errorCode: data?.errorCode as string | undefined, + completedAt: at, + updatedAt: at, + } as WorkflowRun; + runs.set(runId, run); + releaseRunResources(runId); + return { run }; + } + + case 'attr_set': { + const existing = runs.get(runId); + if (!existing) return {}; + const attributes = { ...existing.attributes }; + for (const change of (data?.changes ?? []) as { + key: string; + value: string | null; + }[]) { + if (change.value === null) delete attributes[change.key]; + else attributes[change.key] = change.value; + } + const run = { ...existing, attributes, updatedAt: at } as WorkflowRun; + runs.set(runId, run); + return { run }; + } + + case 'step_created': { + if (!correlationId) return {}; + const step: Step = { + runId, + stepId: correlationId, + stepName: data?.stepName as string, + status: 'pending', + input: data?.input as Uint8Array, + attempt: 0, + createdAt: at, + updatedAt: at, + specVersion: event.specVersion, + }; + steps.set(stepKey(runId, correlationId), step); + return { step }; + } + + case 'step_started': + case 'step_completed': + case 'step_failed': + case 'step_retrying': { + if (!correlationId) return {}; + const key = stepKey(runId, correlationId); + const existing = steps.get(key); + if (!existing) return {}; + const step: Step = + event.eventType === 'step_started' + ? { + ...existing, + status: 'running', + startedAt: existing.startedAt ?? at, + attempt: existing.attempt + 1, + retryAfter: undefined, + updatedAt: at, + } + : event.eventType === 'step_completed' + ? { + ...existing, + status: 'completed', + output: data?.result as Uint8Array, + completedAt: at, + updatedAt: at, + } + : event.eventType === 'step_failed' + ? { + ...existing, + status: 'failed', + error: data?.error as Uint8Array, + completedAt: at, + updatedAt: at, + } + : { + ...existing, + status: 'pending', + error: data?.error as Uint8Array, + retryAfter: data?.retryAfter as Date | undefined, + updatedAt: at, + }; + steps.set(key, step); + return { step }; + } + + case 'hook_created': { + if (!correlationId) return {}; + const token = data?.token as string; + const owningRun = runs.get(runId); + const hook: Hook = { + runId, + hookId: correlationId, + token, + metadata: data?.metadata as Uint8Array | undefined, + ownerId: 'sim-owner', + projectId: 'sim-project', + environment: 'sim', + createdAt: at, + specVersion: event.specVersion, + isWebhook: (data?.isWebhook as boolean) ?? false, + isSystem: (data?.isSystem as boolean) ?? false, + ...(owningRun ? { resumeContext: resumeContextFor(owningRun) } : {}), + }; + hooks.set(correlationId, hook); + tokenOwners.set(token, correlationId); + return { hook }; + } + + // A delivered payload changes no row of its own; the hook is reported + // back so the caller can return it. + case 'hook_received': + return correlationId ? { hook: hooks.get(correlationId) } : {}; + + case 'hook_disposed': { + if (!correlationId) return {}; + disposedHooks.add(correlationId); + const existing = hooks.get(correlationId); + if (existing && tokenOwners.get(existing.token) === correlationId) { + tokenOwners.delete(existing.token); + } + hooks.delete(correlationId); + return {}; + } + + case 'wait_created': { + if (!correlationId) return {}; + const key = waitKey(runId, correlationId); + const wait: Wait = { + waitId: key, + runId, + status: 'waiting', + resumeAt: data?.resumeAt as Date | undefined, + createdAt: at, + updatedAt: at, + specVersion: event.specVersion, + }; + waits.set(key, wait); + return { wait }; + } + + case 'wait_completed': { + if (!correlationId) return {}; + const key = waitKey(runId, correlationId); + const existing = waits.get(key); + if (!existing) return {}; + const wait: Wait = { + ...existing, + status: 'completed', + completedAt: at, + updatedAt: at, + }; + waits.set(key, wait); + return { wait }; + } + + default: + return {}; + } + } + + async function create( + runIdArg: string | null, + data: AnyEventRequest, + params?: CreateEventParams + ): Promise { + // Commit time, for the entity rows the transaction writes. The *event's* + // timestamp comes from its minted position instead β€” see `MintedEvent`. + const now = new Date(nowMs()); + const internal = params as + | (CreateEventParams & SimCreateParams) + | undefined; + // Reassigned only by the two paths that write a *synthetic* event ahead of + // the requested one: the synthetic takes the position minted at the + // boundary (production mints it first, for exactly this ordering) and the + // requested event re-mints so it still sorts after. + let position = internal?.minted ?? mintEvent(); + const resolveData: ResolveData = params?.resolveData ?? 'all'; + const specVersion = data.specVersion ?? SPEC_VERSION_CURRENT; + + let runId: string; + if (data.eventType === 'run_created' && !runIdArg) { + runId = ids.runId(); + } else if (!runIdArg) { + throw new Error('runId is required for non-run_created events'); + } else { + runId = runIdArg; + } + + let currentRun = runs.get(runId); + + // ---- Resilient start --------------------------------------------------- + // A `run_started` carrying creation data may legitimately arrive for a run + // whose `run_created` write failed: `start()` fires both concurrently and + // treats a retryable creation failure as recoverable because the queue + // already accepted the run. Create the run (and a synthetic `run_created`) + // from the queued payload. + if (data.eventType === 'run_started' && !currentRun && data.eventData) { + const seed = data.eventData; + if (seed.deploymentId && seed.workflowName && seed.input !== undefined) { + const synthetic = { + eventType: 'run_created', + runId, + ...position, + specVersion, + eventData: { + deploymentId: seed.deploymentId, + workflowName: seed.workflowName, + input: seed.input, + executionContext: seed.executionContext, + attributes: seed.attributes, + encryptionPublicKey: seed.encryptionPublicKey, + }, + } as Event; + currentRun = applyEvent(synthetic, now).run; + append(synthetic); + // The synthetic took the boundary-minted position, so the `run_started` + // row built below needs a fresh one to sort after it. + position = mintEvent(); + } + } + + if ( + (data.eventType === 'run_failed' || data.eventType === 'attr_set') && + !currentRun + ) { + throw new WorkflowRunNotFoundError(runId); + } + + // ---- Optimistic-concurrency fence ------------------------------------- + // Two independent predicates, both evaluated here and both atomic with the + // append below (there is no await between them and it), mirroring + // workflow-server's handler. The first is a high-water mark; the second is + // a count. They fail on different shapes, and only together do they cover + // both halves of a two-writer race. + if (options.preconditionGuard && params?.stateUpdatedAt !== undefined) { + const marker = externalWriteMarker.get(runId); + if (marker !== undefined && params.stateUpdatedAt < marker) { + throw new PreconditionFailedError( + `Run "${runId}" changed out of band since the caller's snapshot` + ); + } + + // The count guard. `recorded > stateEventCount` means the log holds an + // event at or below the caller's own watermark that the caller never + // loaded: a hole, which the marker comparison above passes by + // construction because the missing event is *older* than the newest one + // the caller did see. A `null` count is indeterminate (the window pruned + // past the snapshot) and is never treated as stale β€” the guard is + // deliberately one-sided. + const stateEventCount = internal?.stateEventCount; + if (options.countGuard && stateEventCount !== undefined) { + const index = runEventIndex.get(runId); + const recorded = index + ? countRecordedAtOrBelow(index, params.stateUpdatedAt) + : null; + if (recorded !== null && recorded > stateEventCount) { + throw new PreconditionFailedError( + `Run "${runId}" holds ${recorded} events at or below the caller's ` + + `watermark, but the caller loaded ${stateEventCount}` + ); + } + } + } + + const createsChildEntity = isChildEntityCreationEvent(data); + const lazyStepStart = + createsChildEntity && data.eventType === 'step_started'; + + // ---- Terminal-run guards ---------------------------------------------- + if (currentRun && isTerminalWorkflowRunStatus(currentRun.status)) { + if ( + data.eventType === 'run_cancelled' && + currentRun.status === 'cancelled' + ) { + // Cancelling an already-cancelled run is idempotent. + const event = append({ + ...data, + runId, + ...position, + specVersion, + } as Event); + return { + event: stripEventDataRefs(clone(event), resolveData), + run: clone(currentRun), + maxEvents: MAX_EVENTS_PER_RUN, + }; + } + if (data.eventType === 'run_started') { + throw new RunExpiredError( + `Workflow run "${runId}" is already in terminal state "${currentRun.status}"` + ); + } + if (isTerminalRunEventType(data.eventType)) { + throw new EntityConflictError( + `Cannot transition run from terminal state "${currentRun.status}"` + ); + } + if (createsChildEntity) { + throw new EntityConflictError( + `Cannot create new entities on run in terminal state "${currentRun.status}"` + ); + } + if (data.eventType === 'attr_set') { + throw new EntityConflictError( + `Cannot set attributes on run in terminal state "${currentRun.status}"` + ); + } + } + + // ---- Step ordering guards --------------------------------------------- + let validatedStep: Step | undefined; + if ( + isStepEventType(data.eventType) && + data.eventType !== 'step_created' && + data.correlationId + ) { + validatedStep = steps.get(stepKey(runId, data.correlationId)); + if (!validatedStep && !lazyStepStart) { + throw new WorkflowWorldError(`Step "${data.correlationId}" not found`); + } + // A lazy `step_started` is the exactly-once create claim for its step: + // if the step already exists, another handler won and this caller must + // not run the body. `EntityConflictError` is what the runtime maps to + // "skipped". + if (lazyStepStart && validatedStep) { + throw new EntityConflictError( + `Step "${data.correlationId}" already created` + ); + } + if (validatedStep) { + if (isTerminalStepStatus(validatedStep.status)) { + throw new EntityConflictError( + `Cannot modify step in terminal state "${validatedStep.status}"` + ); + } + if ( + data.eventType === 'step_started' && + validatedStep.retryAfter && + validatedStep.retryAfter.getTime() > nowMs() + ) { + throw new TooEarlyError( + `Cannot start step "${data.correlationId}": retryAfter timestamp has not been reached yet`, + { + retryAfter: Math.ceil( + (validatedStep.retryAfter.getTime() - nowMs()) / 1000 + ), + } + ); + } + if (currentRun && isTerminalWorkflowRunStatus(currentRun.status)) { + // A terminal run still accepts the terminal write of a step that was + // already running when the run ended β€” that write is how an inline + // step reports back β€” but nothing else. + if (validatedStep.status !== 'running') { + throw new RunExpiredError( + `Cannot modify non-running step on run in terminal state "${currentRun.status}"` + ); + } + } + } + } + + // ---- Hook existence guards -------------------------------------------- + if (isHookEventRequiringExistence(data.eventType) && data.correlationId) { + if (disposedHooks.has(data.correlationId)) { + throw new HookNotFoundError(data.correlationId); + } + if (!hooks.has(data.correlationId)) { + throw new HookNotFoundError(data.correlationId); + } + } + + let event: Event = { + ...data, + runId, + ...position, + specVersion, + } as Event; + + // `run_started`'s eventData is a bootstrap payload for the resilient path + // above, not log content β€” the canonical copy lives on `run_created`. + if (data.eventType === 'run_started' && 'eventData' in event) { + delete (event as Record).eventData; + } + + // ---- Per-event-type validation ---------------------------------------- + // Everything the write path *refuses*. What it does to the entity rows is + // `applyEvent` below β€” the same fold the seed path runs. + switch (data.eventType) { + case 'run_created': { + if (runs.has(runId)) { + throw new EntityConflictError( + `Workflow run "${runId}" already exists` + ); + } + break; + } + + case 'run_started': { + if (currentRun?.status === 'running') { + // Idempotent: a concurrent invocation already started the run. No + // event is appended β€” replay must not see two `run_started`. + return { run: clone(currentRun), maxEvents: MAX_EVENTS_PER_RUN }; + } + break; + } + + case 'step_created': { + if (steps.has(stepKey(runId, data.correlationId))) { + throw new EntityConflictError( + `Step "${data.correlationId}" already created` + ); + } + break; + } + + case 'hook_created': { + const { token } = data.eventData; + const owner = tokenOwners.get(token); + if (owner && owner !== data.correlationId) { + // Someone else holds the token. This is not an error for the + // *caller* β€” the workflow needs to observe it and fail its awaited + // hook β€” so it is journaled as a `hook_conflict` event instead. + const conflict = append({ + eventType: 'hook_conflict', + runId, + eventId: event.eventId, + createdAt: now, + specVersion, + correlationId: data.correlationId, + eventData: { + token, + conflictingRunId: hooks.get(owner)?.runId, + }, + } as Event); + return { + event: stripEventDataRefs(clone(conflict), resolveData), + run: currentRun ? clone(currentRun) : undefined, + }; + } + if (hooks.has(data.correlationId)) { + throw new EntityConflictError( + `Hook "${data.correlationId}" already created` + ); + } + // The hook copies a resume context off its run, so that resuming it + // needs no run read. No run, no hook. + requireRun(runId); + break; + } + + case 'hook_disposed': { + if (disposedHooks.has(data.correlationId)) { + throw new EntityConflictError( + `Hook "${data.correlationId}" already disposed` + ); + } + break; + } + + case 'wait_created': { + if (waits.has(waitKey(runId, data.correlationId))) { + throw new EntityConflictError( + `Wait "${data.correlationId}" already exists` + ); + } + break; + } + + case 'wait_completed': { + const existing = waits.get(waitKey(runId, data.correlationId)); + if (!existing) { + throw new WorkflowWorldError( + `Wait "${data.correlationId}" not found` + ); + } + if (existing.status === 'completed') { + throw new EntityConflictError( + `Wait "${data.correlationId}" already completed` + ); + } + break; + } + } + + // ---- Lazy step creation ------------------------------------------------ + // A `step_started` that carries a payload and finds no step of its own both + // creates and starts one. The synthetic `step_created` keeps replay honest, + // because the client's step consumer only flips `hasCreatedEvent` on that + // event type. + let stepCreatedLazily = false; + if ( + data.eventType === 'step_started' && + lazyStepStart && + !validatedStep && + data.eventData + ) { + const created = { + eventType: 'step_created', + runId, + ...position, + specVersion, + correlationId: data.correlationId, + eventData: { + stepName: data.eventData.stepName, + input: data.eventData.input, + }, + } as Event; + applyEvent(created, now); + append(created); + stepCreatedLazily = true; + + // The input now lives on the synthetic `step_created`; keep only the + // metadata on the `step_started` row. The synthetic took the position + // minted at the boundary β€” production mints it first for this very + // reason β€” so re-mint the `step_started` to sort after it. + // + // Consequence worth knowing: a lazy `step_started` held mid-flight does + // *not* keep an early position, because the pair's positions are settled + // here, at commit. Production mints both in the handler, so it can hold + // an early position for either. No scenario needs that yet; the writes + // that race for position in practice are the completions. + const { input: _dropped, ...rest } = data.eventData; + position = mintEvent(); + event = { ...event, ...position, eventData: rest } as Event; + } + + // ---- The fold ---------------------------------------------------------- + const { run, step, hook, wait } = applyEvent(event, now); + + // Reassigned, not just appended: under `appendOnlyLog` the commit is where + // the position is decided, and everything below β€” the fence's marker, the + // returned row β€” has to speak about the event as it actually landed. + event = append(event); + + // Track externally-originated writes for the precondition fence. A write + // that carries no `stateUpdatedAt` did not come from a replay context, so + // it is exactly the kind of out-of-band change a replaying caller needs to + // be fenced against. + // + // Two details are load-bearing, both copied from workflow-server's + // `recordOutsideEvent`: + // + // - The mark is the event's *own* position time, not the commit instant. It + // has to be the same derivation as the client's `stateUpdatedAt` (the + // position time of its newest loaded event) or a client holding exactly + // this event would compare as older and 412 forever. + // - The write is forward-only. Concurrent out-of-band events can commit out + // of position order β€” the whole subject of these scenarios β€” and letting a + // late-committing older event drag the mark backwards would silently + // disarm the guard for the newer one. + if ( + options.preconditionGuard && + params?.stateUpdatedAt === undefined && + (data.eventType === 'hook_received' || + data.eventType === 'step_completed' || + data.eventType === 'step_failed') + ) { + const previous = externalWriteMarker.get(runId) ?? 0; + externalWriteMarker.set( + runId, + Math.max(previous, event.createdAt.getTime()) + ); + } + + // ---- Optional inline event delta -------------------------------------- + // All three fields or none of them: `EventResult` is a union of a populated + // page and an all-`undefined` one, so they travel together as one object + // rather than three variables the type cannot see are in agreement. + let deltaPage: + | { events: Event[]; cursor: string | null; hasMore: boolean } + | undefined; + + if (data.eventType === 'run_started' && run && !params?.skipPreload) { + const page = paginate(eventsForRun(runId), { + pagination: { limit: 1000, sortOrder: 'asc' }, + getCreatedAt: (e) => e.createdAt, + getId: (e) => e.eventId, + }); + deltaPage = { + events: page.data, + cursor: page.cursor, + hasMore: page.hasMore, + }; + } else if ( + isTerminalStepEventType(data.eventType) && + typeof params?.sinceCursor === 'string' + ) { + const page = paginate(applyWithhold(eventsForRun(runId)), { + pagination: { cursor: params.sinceCursor, sortOrder: 'asc' }, + getCreatedAt: (e) => e.createdAt, + getId: (e) => e.eventId, + }); + deltaPage = { + events: page.data.map((e) => stripEventDataRefs(e, resolveData)), + cursor: page.cursor, + hasMore: page.hasMore, + }; + } + + const result = { + event: stripEventDataRefs(clone(event), resolveData), + run: run ? clone(run) : undefined, + step: step ? clone(step) : undefined, + hook: hook ? clone(hook) : undefined, + wait: wait ? clone(wait) : undefined, + // `as const`: outside a returned literal there is no contextual type to + // keep this from widening to `boolean`, and the field is `true | undefined`. + ...(stepCreatedLazily ? { stepCreated: true as const } : {}), + ...(run ? { maxEvents: MAX_EVENTS_PER_RUN } : {}), + }; + // Spread as a whole or not at all, and as a *conditional* rather than an + // optional spread: the latter widens the three fields to `T | undefined`, + // which is neither arm of the union. + return deltaPage ? { ...result, ...deltaPage } : result; + } + + const storage: SimStore = { + runs: { + async get(id: string, params?: { resolveData?: ResolveData }) { + const run = runs.get(id); + if (!run) throw new WorkflowRunNotFoundError(id); + const copy = clone(run); + if (params?.resolveData === 'none') { + return { ...copy, input: undefined, output: undefined } as never; + } + return copy as never; + }, + async getMany( + idList: readonly string[], + params?: { resolveData?: ResolveData } + ) { + return Promise.all( + idList.map(async (id) => + runs.has(id) ? await storage.runs.get(id, params as never) : null + ) + ) as never; + }, + async list(params?: { + workflowName?: string; + status?: WorkflowRun['status']; + pagination?: PaginationOptions; + resolveData?: ResolveData; + }) { + let items = [...runs.values()]; + if (params?.workflowName) { + items = items.filter((r) => r.workflowName === params.workflowName); + } + if (params?.status) { + items = items.filter((r) => r.status === params.status); + } + const page = paginate(items, { + pagination: params?.pagination, + getCreatedAt: (r) => r.createdAt, + getId: (r) => r.runId, + }); + if (params?.resolveData === 'none') { + return { + ...page, + data: page.data.map((r) => ({ + ...r, + input: undefined, + output: undefined, + })), + } as never; + } + return page as never; + }, + }, + + steps: { + async get( + runId: string, + stepId: string, + params?: { resolveData?: ResolveData } + ) { + const found = steps.get(stepKey(runId, stepId)); + if (!found) throw new WorkflowWorldError(`Step "${stepId}" not found`); + const copy = clone(found); + if (params?.resolveData === 'none') { + return { ...copy, input: undefined, output: undefined } as never; + } + return copy as never; + }, + async list(params: { + runId: string; + pagination?: PaginationOptions; + resolveData?: ResolveData; + }) { + const items = [...steps.values()].filter( + (s) => s.runId === params.runId + ); + const page = paginate(items, { + pagination: params.pagination, + getCreatedAt: (s) => s.createdAt, + getId: (s) => s.stepId, + }); + if (params.resolveData === 'none') { + return { + ...page, + data: page.data.map((s) => ({ + ...s, + input: undefined, + output: undefined, + })), + } as never; + } + return page as never; + }, + }, + + events: { + create: create as Storage['events']['create'], + async get(runId, eventId, params) { + const found = events.find( + (e) => e.runId === runId && e.eventId === eventId + ); + if (!found) + throw new Error(`Event ${eventId} in run ${runId} not found`); + return stripEventDataRefs(clone(found), params?.resolveData ?? 'all'); + }, + async list(params) { + const page = paginate(applyWithhold(eventsForRun(params.runId)), { + pagination: params.pagination, + defaultSortOrder: 'asc', + getCreatedAt: (e) => e.createdAt, + getId: (e) => e.eventId, + }); + const resolve = params.resolveData ?? 'all'; + return { + ...page, + data: page.data.map((e) => stripEventDataRefs(e, resolve)), + }; + }, + async listByCorrelationId(params) { + const page = paginate( + events.filter((e) => e.correlationId === params.correlationId), + { + pagination: params.pagination, + defaultSortOrder: 'asc', + getCreatedAt: (e) => e.createdAt, + getId: (e) => e.eventId, + } + ); + const resolve = params.resolveData ?? 'all'; + return { + ...page, + data: page.data.map((e) => stripEventDataRefs(e, resolve)), + }; + }, + }, + + hooks: { + async get(hookId) { + const found = hooks.get(hookId); + if (!found) throw new HookNotFoundError(hookId); + return clone(found); + }, + async getByToken(token) { + const hookId = tokenOwners.get(token); + const found = hookId ? hooks.get(hookId) : undefined; + if (!found) throw new HookNotFoundError(token); + return clone(found); + }, + async list(params) { + const items = [...hooks.values()].filter( + (h) => !params.runId || h.runId === params.runId + ); + return paginate(items, { + pagination: params.pagination, + getCreatedAt: (h) => h.createdAt, + getId: (h) => h.hookId, + }); + }, + }, + + withholdNextEvent(reads = 1) { + armedWithhold = reads; + }, + + mintEvent, + + seedFromLog(log) { + for (const event of log) { + const seeded = clone(event) as Event; + events.push(seeded); + recordInIndex(seeded); + // The event's own position time is the only clock a seeded row can + // have: the live one belongs to whenever this world was built. + applyEvent(seeded, seeded.createdAt); + } + }, + + // Log order, which is position order β€” not the order the appends happened. + // The two differ exactly when a write was minted before another and + // committed after it, which is the fault these scenarios are about. The + // trace keeps commit order; this is what a reader sees. + // + // Under `appendOnlyLog` the two orders are the same by construction, and + // this sort is a no-op that stays for the invariant it documents. + allEvents: (runId) => + (runId ? eventsForRun(runId) : events) + .map(clone) + .sort( + (a, b) => + a.createdAt.getTime() - b.createdAt.getTime() || + a.eventId.localeCompare(b.eventId) + ), + allEventsInCommitOrder: (runId) => + (runId ? eventsForRun(runId) : events).map(clone), + allRuns: () => [...runs.values()].map(clone), + allSteps: (runId) => + [...steps.values()].filter((s) => !runId || s.runId === runId).map(clone), + allHooks: (runId) => + [...hooks.values()].filter((h) => !runId || h.runId === runId).map(clone), + allWaits: (runId) => + [...waits.values()].filter((w) => !runId || w.runId === runId).map(clone), + hookByToken: (token) => { + const hookId = tokenOwners.get(token); + const found = hookId ? hooks.get(hookId) : undefined; + return found ? clone(found) : undefined; + }, + }; + + return storage; +} diff --git a/packages/world-sim/src/streams.ts b/packages/world-sim/src/streams.ts new file mode 100644 index 0000000000..78e717b6f1 --- /dev/null +++ b/packages/world-sim/src/streams.ts @@ -0,0 +1,160 @@ +/** + * In-memory streamer. + * + * Streams carry step/workflow payloads that exceed the inline event budget, + * plus any `ReadableStream` a workflow passes around. The simulation keeps + * them as plain chunk arrays. + * + * The one subtlety is `get()`: a reader can legitimately attach to a stream + * before the writer has produced anything, and must then observe chunks as + * they arrive rather than seeing an empty stream. Because the scheduler runs + * one delivery at a time and never blocks on wall-clock time, a reader that + * parked on an unfinished stream would deadlock the scenario. So readers park + * on a promise that the *writer* resolves, and `abortOpenReaders()` releases + * any that are still parked when the scenario ends β€” turning what would be a + * hang into a reported diagnostic. + */ + +import type { + GetChunksOptions, + StreamChunksResponse, + Streamer, +} from '@workflow/world'; + +interface StreamState { + chunks: Uint8Array[]; + closed: boolean; + /** Resolvers for readers waiting on more data. */ + waiters: (() => void)[]; +} + +export interface SimStreamer extends Streamer { + /** Number of readers currently parked on an unfinished stream. */ + openReaderCount(): number; + /** Release every parked reader; used at scenario teardown. */ + abortOpenReaders(): void; + streamNames(runId: string): string[]; +} + +function toBytes(chunk: string | Uint8Array): Uint8Array { + return typeof chunk === 'string' ? new TextEncoder().encode(chunk) : chunk; +} + +export function createSimStreamer(): SimStreamer { + const streams = new Map(); + let openReaders = 0; + let aborted = false; + + const key = (runId: string, name: string) => `${runId}\0${name}`; + + function stateFor(runId: string, name: string): StreamState { + const k = key(runId, name); + let state = streams.get(k); + if (!state) { + state = { chunks: [], closed: false, waiters: [] }; + streams.set(k, state); + } + return state; + } + + function wake(state: StreamState) { + const waiters = state.waiters; + state.waiters = []; + for (const w of waiters) w(); + } + + return { + streams: { + async write(runId, name, chunk) { + const state = stateFor(runId, name); + state.chunks.push(toBytes(chunk)); + wake(state); + }, + async writeMulti(runId, name, chunks) { + const state = stateFor(runId, name); + for (const chunk of chunks) state.chunks.push(toBytes(chunk)); + wake(state); + }, + async close(runId, name) { + const state = stateFor(runId, name); + state.closed = true; + wake(state); + }, + async get(runId, name, startIndex = 0) { + const state = stateFor(runId, name); + let index = + startIndex < 0 + ? Math.max(0, state.chunks.length + startIndex) + : startIndex; + openReaders++; + let released = false; + const release = () => { + if (released) return; + released = true; + openReaders--; + }; + return new ReadableStream({ + async pull(controller) { + while (index >= state.chunks.length) { + if (state.closed || aborted) { + release(); + controller.close(); + return; + } + await new Promise((resolve) => state.waiters.push(resolve)); + } + controller.enqueue(state.chunks[index++]); + }, + cancel() { + release(); + }, + }); + }, + async list(runId) { + const prefix = `${runId}\0`; + return [...streams.keys()] + .filter((k) => k.startsWith(prefix)) + .map((k) => k.slice(prefix.length)); + }, + async getChunks( + runId: string, + name: string, + options?: GetChunksOptions + ): Promise { + const state = stateFor(runId, name); + // `Math.max(1, …)`: a caller asking for zero chunks gets one rather + // than an empty page that reports `hasMore` forever. + const limit = Math.max(1, Math.min(options?.limit ?? 100, 1000)); + // A cursor is an opaque string from a previous page; a garbled one + // must not become `NaN` and slice the whole array away silently. + const parsed = Number(options?.cursor ?? 0); + const from = + Number.isFinite(parsed) && parsed > 0 ? Math.floor(parsed) : 0; + const slice = state.chunks.slice(from, from + limit); + const next = from + slice.length; + return { + data: slice.map((data, i) => ({ index: from + i, data })), + cursor: next < state.chunks.length ? String(next) : null, + hasMore: next < state.chunks.length, + done: state.closed, + }; + }, + async getInfo(runId, name) { + const state = stateFor(runId, name); + return { tailIndex: state.chunks.length - 1, done: state.closed }; + }, + }, + + openReaderCount: () => openReaders, + abortOpenReaders() { + aborted = true; + for (const state of streams.values()) wake(state); + }, + streamNames(runId) { + const prefix = `${runId}\0`; + return [...streams.keys()] + .filter((k) => k.startsWith(prefix)) + .map((k) => k.slice(prefix.length)); + }, + }; +} diff --git a/packages/world-sim/src/tempo.test.ts b/packages/world-sim/src/tempo.test.ts new file mode 100644 index 0000000000..c71ec48e9e --- /dev/null +++ b/packages/world-sim/src/tempo.test.ts @@ -0,0 +1,174 @@ +/** + * The imperative scripting layer is the one part of the simulator that can + * hang: a parked call blocks the scheduler, so a script waiting for something + * that never happens has no quiescence to fall back on. These tests pin the + * guards that turn each such case back into a reported failure. + * + * They drive `runScenario` against a handler that acknowledges deliveries + * without doing any work, so the run makes no progress and every wait a script + * could express is one that will never be satisfied. + */ + +import { describe, expect, it } from 'vitest'; +import { runScenario, type ScenarioSpec } from './scenario.js'; + +/** A flow handler that accepts every delivery and advances nothing. */ +const inertHandler = async () => Response.json({ ok: true }); + +const WORKFLOW_ID = 'workflow//./workflows/demo//demoWorkflow'; + +function scenario(partial: Partial): ScenarioSpec { + return { + name: 'test', + workflow: { workflowId: WORKFLOW_ID }, + limits: { maxWallMs: 1_000 }, + ...partial, + }; +} + +describe('tempo scripts', () => { + it('reports a script still waiting instead of hanging', async () => { + const result = await runScenario( + scenario({ + script: async (sim) => { + // Nothing ever commits a step_started against the inert handler. + await sim.park({ eventType: 'step_started' }); + }, + }), + { handler: inertHandler } + ); + + expect(result.ok).toBe(false); + expect(result.problems.join('\n')).toMatch( + /script never finished β€” waiting for "park step_started"/ + ); + // The run itself is diagnosed independently: the queue drained with the + // run still going. + expect(result.outcome).toBe('stalled'); + }); + + it('breaks a parked call on the wall-clock deadline', async () => { + const started = performance.now(); + const result = await runScenario( + scenario({ + limits: { maxWallMs: 250 }, + script: async (sim) => { + // Park on the very first world call and never release it. Without + // the deadline this deadlocks the scheduler permanently. + const parked = await sim.park({ eventType: 'run_created' }); + await new Promise(() => {}); + parked.release(); + }, + }), + { handler: inertHandler } + ); + + expect(result.ok).toBe(false); + expect(performance.now() - started).toBeLessThan(5_000); + expect(result.problems.join('\n')).toMatch(/wall-clock budget/); + }); + + it('runs a script that parks, acts and releases', async () => { + const observed: string[] = []; + const result = await runScenario( + scenario({ + script: async (sim) => { + const parked = await sim.park({ eventType: 'run_created' }); + // The world call has committed but not returned: the event is in + // the log and `start()` has not been resumed. + observed.push( + ...sim.world.events().map((e: { eventType: string }) => e.eventType) + ); + parked.release(); + }, + }), + { handler: inertHandler } + ); + + expect(observed).toEqual(['run_created']); + // No script problems β€” only the run's own stall. + expect(result.problems.filter((p) => p.includes('script'))).toEqual([]); + }); + + it('surfaces a script failure as a scenario problem, not a world error', async () => { + const result = await runScenario( + scenario({ + script: async (sim) => { + const parked = await sim.park({ eventType: 'run_created' }); + parked.release(); + throw new Error('assertion in script'); + }, + }), + { handler: inertHandler } + ); + + expect(result.problems.join('\n')).toMatch( + /script threw: assertion in script/ + ); + // The run was not derailed by the script's failure. + expect(result.outcome).toBe('stalled'); + }); + + it('releases a parked call exactly once, even if the script double-releases', async () => { + const result = await runScenario( + scenario({ + script: async (sim) => { + const parked = await sim.park({ eventType: 'run_created' }); + parked.release(); + parked.release(); + }, + }), + { handler: inertHandler } + ); + + expect(result.problems.filter((p) => p.includes('script'))).toEqual([]); + }); + + it('lets `during` scope the hold to a block', async () => { + let heldEvents = 0; + await runScenario( + scenario({ + script: async (sim) => { + await sim.during({ eventType: 'run_created' }, () => { + heldEvents = sim.world.events().length; + }); + }, + }), + { handler: inertHandler } + ); + expect(heldEvents).toBe(1); + }); +}); + +describe('delivery order', () => { + it('lets a scenario override which message is delivered next', async () => { + const seen: string[] = []; + await runScenario( + scenario({ + selectNext: (pending) => { + seen.push(...pending.map((m) => m.messageId)); + // Deliberately pick the last pending message rather than the first. + return pending.at(-1)?.messageId; + }, + }), + { handler: inertHandler } + ); + // start() enqueues exactly one message, so the override is exercised even + // though the choice is forced. + expect(seen.length).toBeGreaterThan(0); + }); + + it('falls back to the default order when the choice is not pending', async () => { + const result = await runScenario( + scenario({ selectNext: () => 'msg_does_not_exist' }), + { handler: inertHandler } + ); + expect( + result.trace.some( + (t) => t.kind === 'warn' && t.message.includes('is not pending') + ) + ).toBe(true); + // The message still got delivered via the default path. + expect(result.deliveries).toBe(1); + }); +}); diff --git a/packages/world-sim/src/tempo.ts b/packages/world-sim/src/tempo.ts new file mode 100644 index 0000000000..e0c7e1363a --- /dev/null +++ b/packages/world-sim/src/tempo.ts @@ -0,0 +1,262 @@ +/** + * The scripting layer: stop a writer inside a world call, act while it is held, + * let it go. + * + * Everything here compiles down to one thing β€” a watch on a call point whose + * action returns a promise, which blocks the intercepted call until that + * promise settles. `Writer.runTo*` (see `writers.ts`) is the vocabulary a + * scenario should reach for; `park` / `until` / `during` are the primitive it is + * built from, kept public for points no writer op names. + * + * The hazard a script introduces is that it can wait for something that will + * never happen, and because a held call blocks the writer that made it β€” and, + * when that writer is the one the scheduler is inside, the whole loop β€” that is + * a hang rather than a quiescent stall. Three things buy the termination + * guarantee back: + * + * - `runTo` is **level-triggered**: a point that has already gone by is an + * error naming the call it happened at, not a wait. + * - Each `runTo` carries **its own wall-clock budget**, and blowing it reports + * where every writer was standing. + * - The scenario arms a **global wall-clock deadline** that releases every held + * call and rejects every pending wait, as a last resort. + */ + +import type { + CallContext, + CallMatch, + Parked, + ScenarioApi, + Tempo, +} from './types.js'; +import type { SimWorld } from './world.js'; +import { type ArmedHold, createWriters } from './writers.js'; + +/** + * Raised into a script's pending waits when the scenario tears down. It means + * "the world stopped, stop waiting" β€” not a failure of the script β€” so the + * runner does not report it as one. + */ +export class ScenarioAborted extends Error { + override readonly name = 'ScenarioAborted'; +} + +export interface TempoController { + tempo: Tempo; + /** + * Release everything and reject everything still waiting. Called when the + * scenario ends, whether cleanly or on the wall-clock deadline. + */ + abort(reason: string): void; + /** Human-readable description of what the script is still waiting for. */ + describeWaiting(): string | undefined; + /** Whether anything is still parked or waiting. */ + isWaiting(): boolean; +} + +interface PendingWait { + label: string; + reject(err: Error): void; +} + +export function describeMatch(match: CallMatch): string { + const bits: string[] = []; + if (match.eventType) { + bits.push( + Array.isArray(match.eventType) + ? match.eventType.join('|') + : match.eventType + ); + } + if (match.call) { + bits.push(Array.isArray(match.call) ? match.call.join('|') : match.call); + } + if (match.stepName) bits.push(`step=${match.stepName}`); + if (match.correlationId) bits.push(`correlation=${match.correlationId}`); + if (match.token) bits.push(`token=${match.token}`); + if (match.where) bits.push('where(...)'); + if (match.phase) bits.push(match.phase); + return bits.join(' ') || 'any call'; +} + +export interface CreateTempoOptions { + /** Wall-clock budget for a single `runTo` that does not set its own. */ + runToWallMs?: number; +} + +export function createTempo( + world: SimWorld, + api: ScenarioApi, + options: CreateTempoOptions = {} +): TempoController { + /** Waits that have not been satisfied yet. */ + const waiting = new Map(); + /** Calls currently held inside the world. */ + const held = new Map(); + let seq = 0; + let abortedWith: string | undefined; + + /** + * Arm a hold and hand back both the wait and a way to call it off. + * + * The watch is registered *synchronously*, before this returns, which is what + * makes "arm two holds, then await both" work: awaiting the first would yield + * the event loop, and the second writer may sail past its point in that gap. + */ + function armHold(match: CallMatch, label: string): ArmedHold { + const id = seq++; + let settle: ((err: Error) => void) | undefined; + let reached = false; + + const promise = new Promise<{ ctx: CallContext; release(): void }>( + (resolve, reject) => { + if (abortedWith) { + reject(new ScenarioAborted(abortedWith)); + return; + } + settle = reject; + // Indirect through `settle` rather than storing `reject`: `settle` is + // replaced below with the wrapper that also disposes the watch, and + // `abort()` walks this map. Storing the raw `reject` here would reject + // the script's promise while leaving the watch armed β€” and a watch that + // fires after an abort blocks its call on a promise whose `release` is + // no longer reachable from anywhere, which is a hang rather than a + // late wake-up (the global deadline is one-shot and already spent). + waiting.set(id, { label, reject: (err: Error) => settle?.(err) }); + + const dispose = world.addWatch({ + match, + // A script's waits are not "required": a wait that never fires is + // already reported, in more useful detail, as the script still + // waiting or as its own timeout. + options: { nth: 1, label }, + action: (ctx) => { + // Belt and braces with the disposing `settle` above: never block a + // call on behalf of a script that is no longer running to release + // it. Returning undefined lets the call through untouched. + if (abortedWith) return; + reached = true; + waiting.delete(id); + dispose(); + // The call now blocks on this promise. Everything the script does + // before releasing happens inside the intercepted call. + return new Promise((letGo) => { + let released = false; + const release = () => { + if (released) return; + released = true; + held.delete(id); + letGo(); + }; + held.set(id, { label, release }); + resolve({ ctx, release }); + }); + }, + }); + + // Calling off a wait has to remove the watch, not just abandon the + // promise: a watch left armed would later stop a writer that nobody is + // going to release. + settle = (err: Error) => { + if (reached) return; + dispose(); + waiting.delete(id); + reject(err); + }; + } + ); + + return { + reached: promise, + cancel: (reason) => settle?.(reason), + }; + } + + const park: Tempo['park'] = async (match, label) => { + const name = label ?? `park ${describeMatch(match)}`; + const armed = armHold(match, name); + const { ctx, release } = await armed.reached; + return { ctx, release } satisfies Parked; + }; + + const until: Tempo['until'] = (match, label) => { + const name = label ?? `until ${describeMatch(match)}`; + if (abortedWith) return Promise.reject(new ScenarioAborted(abortedWith)); + const id = seq++; + + return new Promise((resolve, reject) => { + // Same disposing-reject shape as `armHold`. A leaked watch here does not + // hang anything β€” this action resolves rather than blocking β€” but it + // still stops a writer for a wait nobody is listening to. + waiting.set(id, { + label: name, + reject: (err: Error) => { + dispose(); + waiting.delete(id); + reject(err); + }, + }); + const dispose = world.addWatch({ + match, + options: { nth: 1, label: name }, + action: (ctx) => { + waiting.delete(id); + dispose(); + resolve(ctx); + }, + }); + }); + }; + + const writers = createWriters({ + world, + arm: armHold, + defaultTimeoutMs: options.runToWallMs ?? 5_000, + }); + + const tempo: Tempo = { + ...api, + // `runId` is a getter on `api`; spreading would freeze its value at + // construction time, before the run exists. + get runId() { + return api.runId; + }, + get world() { + return api.world; + }, + writer: writers.handles, + park, + until, + async during(match, body, label) { + const parked = await park(match, label); + try { + return await body(parked); + } finally { + parked.release(); + } + }, + }; + + return { + tempo, + abort(reason) { + abortedWith = reason; + const error = new ScenarioAborted(reason); + for (const wait of waiting.values()) wait.reject(error); + waiting.clear(); + // Release after rejecting, so a script woken by its own rejection does + // not race the scheduler resuming. + for (const entry of [...held.values()]) entry.release(); + held.clear(); + writers.forgetHolds(); + }, + describeWaiting() { + const parts: string[] = []; + for (const entry of held.values()) parts.push(`holding "${entry.label}"`); + for (const wait of waiting.values()) + parts.push(`waiting for "${wait.label}"`); + return parts.length > 0 ? parts.join('; ') : undefined; + }, + isWaiting: () => waiting.size > 0 || held.size > 0, + }; +} diff --git a/packages/world-sim/src/types.ts b/packages/world-sim/src/types.ts new file mode 100644 index 0000000000..9fd5a8ddb5 --- /dev/null +++ b/packages/world-sim/src/types.ts @@ -0,0 +1,557 @@ +import type { + AnyEventRequest, + Event, + EventType, + Hook, + QueuePayload, + Step, + Wait, + WorkflowRun, +} from '@workflow/world'; + +/** + * Every World method the simulation can be paused on. These names are the + * scheduling vocabulary: the requirement is that the deterministic sequence be + * expressed "from whatever the world api is", so a call point is always + * `(one writer, one of these calls, before|after)`. + */ +export type WorldCallName = + | 'getDeploymentId' + | 'queue' + | 'events.create' + | 'events.get' + | 'events.list' + | 'events.listByCorrelationId' + | 'runs.get' + | 'runs.list' + | 'steps.get' + | 'steps.list' + | 'hooks.get' + | 'hooks.getByToken' + | 'hooks.list' + | 'streams.write' + | 'streams.writeMulti' + | 'streams.close' + | 'streams.get' + | 'streams.getChunks' + | 'streams.getInfo' + | 'streams.list'; + +/** + * The two points a world call can be caught at: entering it, and returning + * from it. + * + * A held `'before'` has decided nothing durable and owns no log position, so a + * write that commits during the hold sorts *ahead* of it. For the other order β€” + * a writer that already owns an earlier slot and has not yet appeared β€” hold the + * write itself with `sim.beginHookDelivery`, which reserves the position that + * `events.create` would have minted and hands the caller the moment in between. + * That gap is not a detail: it is the only way a log can gain an event *behind* + * a position a reader has already read past, which is the one shape no + * high-water-mark fence can see. + */ +export type CallPhase = 'before' | 'after'; + +/** + * Who made a world call. + * + * The simulation's whole subject is concurrent writers to one event log, so + * every intercepted call is attributed to one. There are three kinds: + * + * - `'orchestrator'` β€” the workflow function and the machinery around it: the + * suspension handler committing `step_created` / `step_started` / + * `hook_created` / `wait_created`, the run lifecycle writes, and the event + * log *reads* that decide what to do next. One per queue delivery. + * - `` `step:${shortName}` `` β€” one step body. Several are in flight at once + * inside a single delivery, each an independently advanceable async context, + * each writing its own `step_completed` / `step_failed`. This is the writer + * pair that corrupts a log with no out-of-band event involved at all. + * - `'external'` β€” the scenario itself, standing in for everything a real + * deployment does out of band: a webhook receiver calling `resumeHook`, an + * operator cancelling a run. + * + * Two steps sharing a function name share a writer id. That is a real + * limitation and not worth fixing: a scenario that needs to tell them apart + * should give them distinct names. + * + * The run's very first call β€” the `runs.create` that `start()` makes before any + * workflow code exists β€” is attributed to `'orchestrator'` rather than + * `'external'`. Formally the client is out of band, but a scenario reads + * `wf.runToEventCommitted('run_created')` as "let the run get created", and + * giving that call to a different writer than every other step of the run's own + * progression would be a wart, not a distinction. + */ +export type WriterId = 'orchestrator' | 'external' | (string & {}); + +/** + * What the simulation knows at a call point. + * + * `phase: 'after'` means the call's effect is committed to the store but the + * awaiting caller has not been resumed yet β€” this is the window the + * requirement calls out ("the world will add the hook before returning from + * whatever call commits the step started"). + */ +export interface CallContext { + /** Monotonic index over every intercepted World call in the scenario. */ + seq: number; + call: WorldCallName; + phase: CallPhase; + /** Which writer made this call. */ + writer: WriterId; + args: readonly unknown[]; + /** Virtual time at which the call was observed. */ + atMs: number; + /** The run the call concerns, when it can be determined. */ + runId?: string; + /** For `events.create`: the request as submitted. */ + request?: AnyEventRequest; + /** For `events.create` in the `after` phase: the committed event. */ + event?: Event; + /** For `queue`: the enqueued payload. */ + message?: QueuePayload; + /** Present in the `after` phase of a call that threw. */ + error?: unknown; +} + +/** + * One point a writer reached, recorded whether or not anything was waiting for + * it. This history is what makes `runTo` level-triggered: it can answer "has + * this already happened?" instead of arming a watch that will never fire. + */ +export interface ObservedPoint { + /** + * Position in the recorded history, counting from 0. + * + * `seq` cannot serve: a call is recorded twice, once per phase, and both + * records carry the call's `seq`. A writer held at a call's `after` phase has + * to count that call's `before` phase as behind it, and only a per-record + * ordinal orders the two. + */ + ordinal: number; + seq: number; + writer: WriterId; + call: WorldCallName; + phase: CallPhase; + eventType?: EventType; + /** Short step name, when the call carries one. */ + stepName?: string; + token?: string; + correlationId?: string; + /** + * Nesting depth at which the point occurred. Depth > 0 means it happened + * inside another call the scenario was already inside, where a hold is not + * possible β€” worth distinguishing in an error message. + */ + depth: number; + /** + * Whether the call threw. Only meaningful on an `after` point: a `before` + * point happens while the outcome is still unknown, so it is always `false` + * there. + * + * Recorded so the level-triggered check agrees with `CallMatch.failed`. A + * `runToEventCommitted` that ignored this would count a rejected write as the + * commit it was waiting for β€” routine under the fence, where a 412 is an + * expected step on the way to a successful retry. + */ + failed: boolean; +} + +/** An intercepted world call that threw. */ +export interface RejectedCall { + seq: number; + call: WorldCallName; + writer: WriterId; + eventType?: EventType; + /** Error constructor name, e.g. `PreconditionFailedError`. */ + errorName: string; + message: string; +} + +/** Read-only view of world state, for matchers and assertions. */ +export interface WorldSnapshot { + nowMs(): number; + runs(): WorkflowRun[]; + run(runId: string): WorkflowRun | undefined; + events(runId?: string): Event[]; + steps(runId?: string): Step[]; + hooks(runId?: string): Hook[]; + waits(runId?: string): Wait[]; + /** Queue messages that have been enqueued but not yet delivered. */ + pendingMessages(): PendingMessageView[]; + /** + * Every intercepted world call that threw, in order. + * + * Rejections are the visible mechanism behind a run that self-corrects β€” a + * `PreconditionFailedError` from the optimistic-concurrency fence, an + * `EntityConflictError` from a write against an already-terminal run β€” so + * they are recorded unconditionally rather than left to a scenario to + * instrument. + */ + rejections(): RejectedCall[]; +} + +export interface PendingMessageView { + messageId: string; + queueName: string; + runId?: string; + stepId?: string; + /** Virtual time at which the message becomes deliverable. */ + readyAtMs: number; + /** How many times it has already been handed to a handler. */ + deliveries: number; +} + +/** + * Which call point to wait at. Every provided field must match (AND). + * `where` runs last and can inspect the whole world. + * + * This is the low-level vocabulary; scenarios normally express a point through + * a `Writer` (`wf.runToEventCommitted('step_started', 'reserveInventory')`) + * rather than assembling a match by hand. + */ +export interface CallMatch { + call?: WorldCallName | WorldCallName[]; + /** + * Which side of the call to match. Defaults to `'after'` β€” the window where + * the effect is committed but the caller has not been resumed, which is the + * one worth injecting into. Set `'before'` to act ahead of the write. + */ + phase?: CallPhase; + /** Shorthand for `call: 'events.create'` restricted to these event types. */ + eventType?: EventType | EventType[]; + /** Matches `eventData.stepName`, by exact value or by short name suffix. */ + stepName?: string; + /** Matches the event's `correlationId`. */ + correlationId?: string; + /** Matches the hook token on `hook_created` / `hook_received` events. */ + token?: string; + runId?: string; + /** Restrict to calls made by a particular writer. */ + writer?: WriterId | ((writer: WriterId) => boolean); + /** Only match calls that threw (`true`) or succeeded (`false`). */ + failed?: boolean; + where?: (ctx: CallContext, world: WorldSnapshot) => boolean; +} + +// --------------------------------------------------------------------------- +// Writers +// --------------------------------------------------------------------------- + +/** Extra conditions on a `runTo` point. */ +export interface RunToOptions { + /** Short step name carried by the event, e.g. `'reserveInventory'`. */ + stepName?: string; + /** Hook token, for `hook_created` / `hook_received`. */ + token?: string; + correlationId?: string; + /** + * Extra condition on world state, evaluated at the candidate point. Use it + * for "once two steps have completed" β€” a condition about the world rather + * than about one event. + */ + where?: (world: WorldSnapshot) => boolean; + /** Label for the trace. Defaults to a description of the point. */ + label?: string; + /** Wall-clock budget for this one wait. Defaults to `limits.maxRunToWallMs`. */ + timeoutMs?: number; +} + +/** A writer stopped at a point, waiting to be let go. */ +export interface Held { + /** The writer actually caught β€” concrete even when the handle was `anyStep()`. */ + writer: WriterId; + /** What the world was asked to do, and (once committed) what it did. */ + ctx: CallContext; + /** + * Let the writer continue. Idempotent. Awaiting it yields the event loop, so + * once it resolves the released writer has actually made progress. + */ + release(): Promise; +} + +/** + * One writer, steerable a point at a time. + * + * `runTo*` advances this writer to its next matching point and *stops it + * there*, with every other writer free to keep going. It is level-triggered: + * if the point has already gone by, it throws with the seq it happened at + * rather than arming a watch that can never fire, because a missed edge in a + * world where a held call blocks the scheduler is a hang, not a late wakeup. + * + * The corollary is that holds must be **armed before they are needed**. To hold + * two writers at the same point, start both waits and then await them: + * + * ```ts + * const atFast = fast.runToEventProduced('step_completed'); // armed here + * const atSlow = slow.runToEventProduced('step_completed'); // and here + * await atFast; + * await atSlow; + * ``` + * + * Awaiting the first before starting the second yields the event loop, and the + * other writer may well sail past its point in that gap. + */ +export interface Writer { + readonly id: WriterId; + /** + * Stop once the event has crossed the world boundary β€” fully formed, + * attributed to this writer, already in the trace β€” and before it is assigned + * a position in the event log. + * + * Having no position yet, a write that commits to storage while this one is + * held sorts *ahead* of it. For the opposite β€” an event that already owns an + * earlier slot and has not appeared β€” hold the write itself with + * `sim.beginHookDelivery`, which reserves the position on one side of the + * gap and commits on the other. + */ + runToEventProduced( + eventType: EventType | EventType[], + options?: string | RunToOptions + ): Promise; + /** + * Stop just *after* the event is durable and before the writer is resumed. + * This is the window the requirement is about: "receive the hook after a + * step_started is committed but before the workflow resumes running". + */ + runToEventCommitted( + eventType: EventType | EventType[], + options?: string | RunToOptions + ): Promise; + /** Let this writer go, if it is held. Idempotent. */ + release(): Promise; + isHeld(): boolean; + /** Points this writer has reached, oldest first. */ + history(): readonly ObservedPoint[]; +} + +/** + * Handles onto the writers a scenario can steer. + * + * A handle is a *name*, not a live object: `sim.writer.step('slow')` can be + * taken before that step exists and resolves against whichever writer shows up + * under the name. + */ +export interface WriterHandles { + /** The workflow function and the runtime machinery around it. */ + orchestrator(): Writer; + /** One step body, by short function name. */ + step(shortName: string): Writer; + /** Whichever step body reaches the point first. */ + anyStep(): Writer; + /** Any writer at all. */ + any(): Writer; + /** Writer ids seen so far, in first-appearance order. */ + seen(): WriterId[]; +} + +/** + * Everything the scenario script can do to the world. + * + * These are the only sanctioned sources of external input. Anything a real + * deployment could do out-of-band β€” a webhook arriving, an operator + * cancelling a run, time passing β€” has an entry here, so the scenario script + * is a complete description of what happened. + */ +export interface ScenarioApi { + /** Read-only view of the world at this instant. */ + world: WorldSnapshot; + /** + * Deliver a hook payload, exactly as an out-of-band `resumeHook()` would: + * commit `hook_received` and enqueue the run's flow message. + * + * Called while a writer is held, this happens *inside* the call that writer + * is stopped at, so the event lands in the log before that writer is resumed. + * That is the entire point of the writer API. + */ + deliverHook(token: string, payload: unknown): Promise; + /** + * Start a hook delivery and stop between its two halves: the log position is + * taken, the event is not there yet. + * + * This is the fault the `in-flight` scenarios are about, and it needs no stale + * read to express. The receiver's write has entered the handler, so its event + * id β€” the log's sort key β€” is fixed; it has not reached storage, so every + * reader sees a complete, consistent log that simply does not contain it. When + * `commit()` runs, the event appears *behind* positions those readers already + * read past, which is the one shape a high-water mark cannot represent. + * + * Unlike a held writer, nothing is blocked in the meantime: the receiver is a + * separate process from the run's invocation. Holding an *inline* write instead + * would stall the delivery that made it, and thus the reader too β€” which is why + * the out-of-band writer is the one that can do this. + */ + beginHookDelivery(token: string, payload: unknown): Promise; + /** Cancel the run under test, as an operator would. */ + cancelRun(reason?: string): Promise; + /** Jump virtual time forward. */ + advanceTime(ms: number): void; + /** + * Deliver one pending queue message now, jumping the clock to its ready + * time, without waiting for the delivery loop to reach it. + * + * This is the timer counterpart of {@link deliverHook}, and it exists for + * one reason: the delivery loop is serial, so while a script holds an inline + * step body the loop is stopped inside that same delivery and no timer can + * fire. Every interleaving in which a `wait_completed` lands *while a step + * result is still outstanding* is therefore unreachable from the loop alone + * β€” and that is not an exotic corner, it is what + * `Promise.race([step, sleep])` does whenever the step is slower than the + * sleep. + * + * Calling this runs a second flow delivery concurrently with the held one, + * which is what a real queue does with two messages for the same run. + * Concurrency in this simulator is otherwise structural rather than + * scheduled, so this is the one place a script creates some; it stays + * deterministic because the script decides both when it starts and β€” through + * the writer it is holding β€” when the other delivery resumes. + * + * `select` receives the pending messages in the loop's own order β€” earliest + * `readyAt`, then enqueue order β€” and returns a `messageId`. The default + * takes the first, i.e. exactly what the loop would have done next. A script + * that wants a timer specifically should say so rather than rely on the + * default: a hook delivery enqueues a flow message too, and it will usually + * be earlier. Resolves `false` when nothing matched, so a script can assert + * that something fired. + */ + deliverQueued( + select?: (pending: PendingMessageView[]) => string | undefined + ): Promise; + /** + * Hide the next event this scenario commits from the following `reads` + * event-log reads, modelling one concurrent writer the reader missed. + * + * Call it immediately before the write you want hidden. This is the only way + * a serial simulation can produce "a write derived from an incomplete event + * load" β€” the precondition a real deployment reaches through concurrency. + */ + withholdNextEvent(reads?: number): void; + /** Record a free-text marker in the scenario trace. */ + note(message: string): void; + /** Record a named assertion in the trace; a false value fails the scenario. */ + check(name: string, condition: boolean): void; + /** The run under test. */ + runId: string; + /** + * Which log this run is playing against, resolved from the spec and the run + * option. See `ScenarioSpec.appendOnlyLog`. + * + * A check's *name* is a sentence in the trace, and several of them are + * sentences about where a position came from β€” "the hook owns a log position + * but is nowhere in the log" is true of a reserved mint and false of an + * append-only commit. Read this to phrase the sentence for the world the run + * is actually in, rather than narrating one world while playing the other. + * + * It is not for branching the *tempo*. A scenario whose script takes a + * different path under the flag is two scenarios wearing one id, and the diff + * between the two runs stops meaning anything. + */ + appendOnlyLog: boolean; +} + +/** An out-of-band write that owns a log position and has not committed. */ +export interface InFlightWrite { + /** The position it will occupy, whenever it lands. */ + eventId: string; + /** Let it reach storage. */ + commit(): Promise; +} + +/** + * A world call stopped mid-flight, waiting for the script to let it go. The + * low-level form of `Held`, without the writer attribution. + * + * The vocabulary is borrowed from Python's `blanket`, which does the same + * thing for `threading` primitives: the call *parks*, the script issues the + * *permit*, and the resulting order of permits is the scenario's *tempo*. + */ +export interface Parked { + /** What the world was asked to do, and (in the `after` phase) what it did. */ + ctx: CallContext; + /** Let the call return. Idempotent. */ + release(): void; +} + +/** + * What a scenario script is handed. + * + * The scenario vocabulary is writers: name them, advance them one point at a + * time, and the interleaving is the script's control flow rather than a race. + * `park` / `until` / `during` are the primitive underneath β€” reach for them for + * a point no writer op names, such as a plain world *read*. + */ +export interface Tempo extends ScenarioApi { + /** Handles onto the writers this scenario can steer. */ + writer: WriterHandles; + /** + * Wait until a world call reaches a matching point, and hold it there. + * + * Everything that writer would go on to do is suspended while the call is + * parked: the caller is blocked inside the world. That is the point β€” + * whatever the script does next is guaranteed to land before the call + * returns. + * + * Edge-triggered, unlike `runTo`: if the point has already gone by, this + * waits for the next one and reports a hang if there is none. + */ + park(match: CallMatch, label?: string): Promise; + /** Wait for a matching call to happen, without holding it. */ + until(match: CallMatch, label?: string): Promise; + /** Park a call, run `body` while it is held, then release it. */ + during( + match: CallMatch, + body: (parked: Parked) => T | Promise, + label?: string + ): Promise; +} + +/** + * A scenario body. Runs concurrently with the delivery loop, starting before + * the run does so it can hold the very first world call. + */ +export type ScenarioScript = (sim: Tempo) => void | Promise; + +/** One line of the scenario trace β€” either a world event or a simulation action. */ +export type TraceEntry = + | { + kind: 'event'; + seq: number; + atMs: number; + event: Event; + /** Which writer committed it. */ + writer?: WriterId; + /** Depth > 0 means the event was committed from inside another call. */ + depth: number; + } + | { + kind: 'hold'; + seq: number; + atMs: number; + label: string; + /** The call point the writer was stopped at. */ + inside: string; + writer?: WriterId; + depth: number; + } + | { + kind: 'note' | 'delivery' | 'warn'; + seq: number; + atMs: number; + message: string; + depth: number; + } + | { + kind: 'check'; + seq: number; + atMs: number; + name: string; + ok: boolean; + depth: number; + }; + +export interface InvariantViolation { + /** Stable identifier, e.g. `step.no-restart-after-terminal`. */ + rule: string; + message: string; + runId?: string; + eventId?: string; +} diff --git a/packages/world-sim/src/world.ts b/packages/world-sim/src/world.ts new file mode 100644 index 0000000000..2579319a9c --- /dev/null +++ b/packages/world-sim/src/world.ts @@ -0,0 +1,846 @@ +/** + * The simulation World. + * + * Three things make it different from a normal World implementation: + * + * 1. **Every method is a call point.** Each call is wrapped so a scenario can + * stop it `before` it starts and `after` its effect is committed but + * *before* the awaiting caller is resumed. That second window is the whole + * reason this package exists: it is what lets a scenario say "the hook + * arrives after `step_started` is durable and before the workflow gets + * control back" and have it be a fact rather than a race it won by luck. + * + * 2. **Every call is attributed to a writer.** The orchestrator, each step + * body, and the scenario acting from outside are separately named and + * separately steerable, because "several writers appending to one log with + * no serializable isolation between them" is the property under test. + * + * 3. **Nothing happens on its own.** The queue only records messages and the + * clock only moves when the scheduler moves it, so the sequence of world + * calls is a pure function of the scenario. + * + * Watches do not fire for calls made from inside another watch's action. + * Without that rule, a watch on `events.create` would re-trigger on the + * `hook_received` it just wrote, and any scenario using `deliverHook` would + * recurse forever. + */ + +import { AsyncLocalStorage } from 'node:async_hooks'; +import { + type Event, + getQueueTopicPrefix, + type QueuePayload, + SPEC_VERSION_CURRENT, + type World, +} from '@workflow/world'; +import { createVirtualClock, type VirtualClock } from './clock.js'; +import { createIdFactory, type IdFactory, ulidTimeOf } from './ids.js'; +import { createSimQueue, type DirectHandler, type SimQueue } from './queue.js'; +import { createSimStore, type MintedEvent, type SimStore } from './store.js'; +import { createSimStreamer, type SimStreamer } from './streams.js'; +import type { + CallContext, + CallMatch, + ObservedPoint, + RejectedCall, + ScenarioApi, + TraceEntry, + WorldCallName, + WorldSnapshot, + WriterId, +} from './types.js'; + +export const WORKFLOW_QUEUE_PREFIX = getQueueTopicPrefix('workflow'); + +/** + * A one-shot (or repeating) callback attached to a call point. Internal: a + * scenario expresses points through writers, and `Watch` is what those compile + * down to. + */ +export interface Watch { + match: CallMatch; + action: (ctx: CallContext, api: ScenarioApi) => void | Promise; + options: { + /** Fire on the nth match only (1-based). Defaults to 1. */ + nth?: number; + /** Fire on every match instead of just one. Overrides `nth`. */ + every?: boolean; + label?: string; + }; +} + +/** + * How many call points to remember for level-triggered waiting. Generous: a + * scenario runs a few hundred calls, and the bound exists only so a runaway + * workflow cannot grow the array without limit before its budget stops it. + */ +const MAX_HISTORY = 20_000; + +export interface SimWorldOptions { + clock?: VirtualClock; + deploymentId?: string; + /** See `SimStoreOptions.preconditionGuard`. */ + preconditionGuard?: boolean; + /** + * Also enforce the count half of the fence (see `SimStoreOptions.countGuard`), + * and supply `stateEventCount` for the writes the runtime does not count. + * + * Production arms both halves. Since #3145 `@workflow/core` sends + * `stateEventCount` on every replay-context create + * (`preconditionSnapshotParams`), gated only by the + * `WORKFLOW_PRECONDITION_GUARD` kill-switch, and workflow-server's count guard + * is on by default β€” so this tracks `preconditionGuard` rather than being + * opted into per scenario. A run with the fence on and the count off would be + * a world that exists nowhere. + */ + countGuard?: boolean; + /** + * Assign log positions at commit rather than at the handler boundary, so the + * log is append-only and no read can be contradicted by a later one. See + * `SimStoreOptions.appendOnlyLog` for what that buys and what it costs. + * + * The boundary mint still happens β€” `reservePosition` and everything a + * scenario hangs off it work unchanged. It just stops being binding: a held + * write that nothing overtook keeps the position it reserved, and one that was + * overtaken re-mints when it lands. + */ + appendOnlyLog?: boolean; +} + +export interface SimWorld extends World { + clock: VirtualClock; + ids: IdFactory; + store: SimStore; + simQueue: SimQueue; + streamer: SimStreamer; + snapshot: WorldSnapshot; + trace: TraceEntry[]; + + registerHandler(prefix: string, handler: DirectHandler): void; + /** Attach a callback to a call point. Returns a disposer. */ + addWatch(watch: Watch): () => void; + /** Failures thrown by watch actions; swallowed at the call site, reported here. */ + watchErrors(): string[]; + /** Supply the API object handed to watch actions. */ + setScenarioApi(resolve: () => ScenarioApi): void; + /** + * Run `fn` as the scenario acting from outside the run: attributed to the + * `external` writer, and not itself a call point. + */ + asExternal(fn: () => Promise): Promise; + /** + * Take a log position now, to be used by a write that happens later. + * + * The scenario's own calls are not call points (see `fireWatches`), so a script + * cannot hold *itself* between minting and committing the way it holds a + * writer. This pair is how it states the same thing directly: reserve the + * position, do whatever should observe the log without it, then run the write + * inside `withReservedPosition` so it lands where it was reserved. + */ + reservePosition(): MintedEvent; + /** Run `fn` with the next `events.create` taking `position` instead of minting. */ + withReservedPosition( + position: MintedEvent, + fn: () => Promise + ): Promise; + pushTrace(entry: TraceInput): void; + /** Total intercepted world calls so far. */ + callCount(): number; + /** Every call point reached so far, in order. Backs level-triggered waiting. */ + callHistory(): readonly ObservedPoint[]; + /** Every intercepted call that threw. */ + rejections(): readonly RejectedCall[]; +} + +/** A trace entry without the fields the world stamps on (seq, time, depth). */ +type TraceInput = + | { kind: 'event'; event: Event; writer?: WriterId } + | { kind: 'hold'; label: string; inside: string; writer?: WriterId } + | { kind: 'note' | 'delivery' | 'warn'; message: string } + | { kind: 'check'; name: string; ok: boolean }; + +/** Any async world method. */ +type AsyncFn = (...args: never[]) => Promise; + +export function createSimWorld(options: SimWorldOptions = {}): SimWorld { + const clock = options.clock ?? createVirtualClock(); + const ids = createIdFactory(() => clock.now()); + const deploymentId = options.deploymentId ?? 'dpl_sim'; + + const trace: TraceEntry[] = []; + /** + * Whether *this* call chain is scenario-originated: set inside an `asExternal` + * block, i.e. the scenario is the one calling, so the call is attributed to + * `external` and is not itself a call point. + * + * Async-context-scoped rather than a plain counter, because `asExternal` + * brackets whole operations β€” `scenario.ts` wraps all of `resumeHook`, which + * spans several awaits. A counter is a global flag for that whole window, so + * a step body committing concurrently gets read as the scenario's own call: + * attributed `external`, skipped by `fireWatches` (a `runTo` armed on it waits + * out its 5s watchdog instead), and left out of the count guard's loaded-set + * bookkeeping. `attr-from-step-body` showed this as `ext` on the `probe` + * step's own `step_completed`, in the one scenario whose subject is the + * writer column. + * + * Deliberately *not* set for the duration of a watch action β€” see + * `fireWatches`. A held action outlives the call it fired from, and a flag + * held that long would silence every other writer. + */ + const externalCtx = new AsyncLocalStorage(); + const isExternal = () => externalCtx.getStore() === true; + /** Set by `withReservedPosition`; consumed by the next `events.create`. */ + let reservedPosition: MintedEvent | undefined; + let callSeq = 0; + let traceSeq = 0; + + const history: ObservedPoint[] = []; + const rejected: RejectedCall[] = []; + + const pushTrace = (entry: TraceInput): void => { + trace.push({ + ...entry, + seq: traceSeq++, + atMs: clock.now(), + depth: isExternal() ? 1 : 0, + } as TraceEntry); + }; + + const store = createSimStore({ + now: () => clock.now(), + ids, + preconditionGuard: options.preconditionGuard, + countGuard: options.countGuard, + appendOnlyLog: options.appendOnlyLog, + // Fires synchronously inside `events.create`, so it runs in that call's + // async context and `isExternal()` still describes who is writing. + onEvent: (event) => + pushTrace({ kind: 'event', event, writer: writerOfEvent(event) }), + // Two different faults, and the trace should not blur them: one read + // around a committed event, the other stopped before it. + onStaleRead: ({ eventId, hidden, truncated }) => + pushTrace({ + kind: 'warn', + message: truncated + ? `lagging read: log cut short at ${eventId}; ${hidden} committed event(s) not yet visible` + : `stale read: committed event ${eventId} withheld from this event-log read`, + }), + }); + + const simQueue = createSimQueue({ + now: () => clock.now(), + ids, + deploymentId, + }); + const streamer = createSimStreamer(); + + const snapshot: WorldSnapshot = { + nowMs: () => clock.now(), + runs: () => store.allRuns(), + run: (runId) => store.allRuns().find((r) => r.runId === runId), + events: (runId) => store.allEvents(runId), + steps: (runId) => store.allSteps(runId), + hooks: (runId) => store.allHooks(runId), + waits: (runId) => store.allWaits(runId), + pendingMessages: () => simQueue.view(), + rejections: () => [...rejected], + }; + + // ------------------------------------------------------------------------- + // Writer attribution + // ------------------------------------------------------------------------- + + /** `step//./workflows/orders//reserveInventory` -> `reserveInventory`. */ + const shortStepName = (name: string): string => { + const cut = name.lastIndexOf('//'); + return cut === -1 ? name : name.slice(cut + 2); + }; + + /** Resolve a stepId (== the event's correlationId) to its short name. */ + const stepNameOf = ( + runId: string | undefined, + stepId: string | undefined + ): string | undefined => { + if (!stepId) return undefined; + const step = store + .allSteps(runId) + .find((candidate) => candidate.stepId === stepId); + return step ? shortStepName(step.stepName) : undefined; + }; + + /** + * Which writer is responsible for an event. + * + * Everything the *scenario* does is `external` β€” that check comes first, + * because a `run_cancelled` from an operator and a `run_cancelled` from the + * runtime are the same event type written by very different writers, and only + * the call stack can tell them apart. + * + * Otherwise: a step's own result events belong to that step body, an + * attribute write names its writer explicitly in the event, and everything + * else β€” the step and hook and wait *creations*, the run lifecycle β€” is the + * orchestrator committing at a suspension point. + */ + function writerOfEvent(event: { + eventType: string; + runId?: string; + correlationId?: string; + eventData?: unknown; + }): WriterId { + if (isExternal()) return 'external'; + + const data = event.eventData as + | { + stepName?: string; + writer?: { type?: string; stepId?: string }; + } + | undefined; + + switch (event.eventType) { + case 'step_completed': + case 'step_failed': + case 'step_retrying': { + const name = + (data?.stepName && shortStepName(data.stepName)) || + stepNameOf(event.runId, event.correlationId); + return name ? `step:${name}` : 'step:?'; + } + case 'attr_set': { + // The only event that states its writer outright. + if (data?.writer?.type !== 'step') return 'orchestrator'; + const name = stepNameOf(event.runId, data.writer.stepId); + return name ? `step:${name}` : 'step:?'; + } + default: + return 'orchestrator'; + } + } + + /** Which writer is making a world call. */ + function writerOfCall( + call: WorldCallName, + runId: string | undefined, + request: CallContext['request'] + ): WriterId { + if (isExternal()) return 'external'; + if (call !== 'events.create' || !request) return 'orchestrator'; + return writerOfEvent({ ...request, runId }); + } + + const watches: { watch: Watch; matches: number }[] = []; + const watchErrors: string[] = []; + let resolveApi: (() => ScenarioApi) | undefined; + + function matches(watch: Watch, ctx: CallContext): boolean { + const { match } = watch; + + // An unspecified phase means `after`, never "both". Matching both would + // fire every watch twice β€” and, worse, fire a `nth: 1` watch at `before`, + // where the effect it is keyed on has not happened yet and `ctx.event` is + // absent. `before` is the case you opt into. + if ((match.phase ?? 'after') !== ctx.phase) return false; + + if (match.writer !== undefined) { + const ok = + typeof match.writer === 'function' + ? match.writer(ctx.writer) + : match.writer === ctx.writer; + if (!ok) return false; + } + + const eventTypes = match.eventType + ? Array.isArray(match.eventType) + ? match.eventType + : [match.eventType] + : undefined; + + if (eventTypes) { + // `eventType` implies the create call, so a scenario never has to spell + // out `call: 'events.create'` alongside it. + if (ctx.call !== 'events.create') return false; + const type = ctx.event?.eventType ?? ctx.request?.eventType; + if (!type || !eventTypes.includes(type)) return false; + } else if (match.call) { + const calls = Array.isArray(match.call) ? match.call : [match.call]; + if (!calls.includes(ctx.call)) return false; + } + + if (match.runId && ctx.runId !== match.runId) return false; + + if (match.correlationId) { + const correlationId = + ctx.event?.correlationId ?? ctx.request?.correlationId; + if (correlationId !== match.correlationId) return false; + } + + if (match.stepName) { + const data = (ctx.event ?? ctx.request) as + | { eventData?: { stepName?: string } } + | undefined; + const stepName = data?.eventData?.stepName; + // Accept both the machine name (`step//./workflows/x//reserve`) and the + // short function name a scenario author would actually type. + if ( + !stepName || + (stepName !== match.stepName && + !stepName.endsWith(`//${match.stepName}`)) + ) { + return false; + } + } + + if (match.token) { + const data = (ctx.event ?? ctx.request) as + | { eventData?: { token?: string } } + | undefined; + if (data?.eventData?.token !== match.token) return false; + } + + if ( + match.failed !== undefined && + match.failed !== (ctx.error !== undefined) + ) { + return false; + } + + if (match.where) { + // A predicate that throws must not become a world-call failure β€” see the + // note on watch actions below. Treat it as "did not match" and report. + try { + if (!match.where(ctx, snapshot)) return false; + } catch (err) { + watchErrors.push( + `where(...) for "${watch.options.label ?? 'watch'}" threw: ${ + err instanceof Error ? err.message : String(err) + }` + ); + return false; + } + } + + return true; + } + + async function fireWatches(ctx: CallContext): Promise { + // Calls the scenario itself makes are not call points: they would otherwise + // trip the very watches they were made from inside of. + if (isExternal()) return; + + // Iterate a copy: an action may dispose its own watch, or arm a new one. + for (const entry of [...watches]) { + if (!watches.includes(entry)) continue; + if (!matches(entry.watch, ctx)) continue; + entry.matches++; + const opts = entry.watch.options; + if (!opts.every && entry.matches !== (opts.nth ?? 1)) continue; + + pushTrace({ + kind: 'hold', + label: opts.label ?? 'watch', + inside: `${ctx.call}:${ctx.phase}`, + writer: ctx.writer, + }); + + // The action is NOT run inside the external context, and that is + // deliberate. A hold's action does not return until the scenario releases + // it, so raising the depth for the duration would mean: for as long as one + // writer is held, every *other* writer's call stops being a call point and + // every event it commits is attributed to the scenario. Holding one step + // body would make its sibling both invisible and unsteerable β€” the exact + // interleaving the writer vocabulary exists to state. Scenario-originated + // writes get their attribution from `asExternal` instead, which follows + // the call chain rather than the wall clock. + try { + if (!resolveApi) { + throw new Error( + 'A watch fired before the scenario API was installed' + ); + } + await entry.watch.action(ctx, resolveApi()); + } catch (err) { + // A throwing watch must not become a world-call failure. Propagating it + // would make `events.create` reject and send the runtime down its + // error-recovery path, which would then be blamed on the runtime + // rather than on the scenario that actually broke. + const message = err instanceof Error ? err.message : String(err); + watchErrors.push( + `watch "${opts.label ?? 'watch'}" threw inside ${ctx.call}:${ctx.phase}: ${message}` + ); + pushTrace({ + kind: 'warn', + message: `watch "${opts.label ?? 'watch'}" failed: ${message}`, + }); + } + } + } + + /** Best-effort extraction of the run a call concerns, for cue matching. */ + function runIdOf( + call: WorldCallName, + args: readonly unknown[] + ): string | undefined { + switch (call) { + case 'events.create': + case 'events.get': + case 'steps.get': + case 'runs.get': + case 'streams.write': + case 'streams.writeMulti': + case 'streams.close': + case 'streams.get': + case 'streams.getChunks': + case 'streams.getInfo': + case 'streams.list': + return typeof args[0] === 'string' ? args[0] : undefined; + case 'events.list': + case 'steps.list': + case 'hooks.list': + return (args[0] as { runId?: string } | undefined)?.runId; + case 'queue': + return (args[1] as { runId?: string } | undefined)?.runId; + default: + return undefined; + } + } + + /** Remember a call point, so a later `runTo` can tell "already passed". */ + function record(ctx: CallContext): void { + if (history.length >= MAX_HISTORY) return; + const data = (ctx.event ?? ctx.request) as + | { eventData?: { stepName?: string; token?: string } } + | undefined; + const stepName = data?.eventData?.stepName; + history.push({ + ordinal: history.length, + seq: ctx.seq, + writer: ctx.writer, + call: ctx.call, + phase: ctx.phase, + depth: isExternal() ? 1 : 0, + failed: ctx.error !== undefined, + ...(ctx.event?.eventType || ctx.request?.eventType + ? { eventType: ctx.event?.eventType ?? ctx.request?.eventType } + : {}), + ...(stepName ? { stepName: shortStepName(stepName) } : {}), + ...(data?.eventData?.token ? { token: data.eventData.token } : {}), + ...((ctx.event?.correlationId ?? ctx.request?.correlationId) + ? { + correlationId: + ctx.event?.correlationId ?? ctx.request?.correlationId, + } + : {}), + }); + } + + /** + * Which events the log the *runtime* is holding contains, keyed by run. + * + * This reconstructs the array the client has in memory, because the count + * guard compares against that array and nothing else: `stateEventCount` is + * defined as the number of loaded events whose ULID time is at or below + * `stateUpdatedAt` β€” and since `stateUpdatedAt` *is* the maximum of those + * times, that is the whole array. The pair "I loaded N events, the newest at + * T" is what lets the world spot a hole *behind* T, which no comparison + * against T alone can see. + * + * Keyed by run, not by writer: the orchestrator and the inline step bodies of + * one delivery are sim-level writers, but they are one process sharing one + * loaded log, and that log is what the count describes. The out-of-band + * writer is the exception β€” a different process with its own log β€” so its + * calls are excluded, by the same `isExternal()` rule that keeps them from + * being call points. + * + * Everything a caller's own write appends counts as loaded, including events + * the write produces as a side effect (a `step_started` claim also appends + * the `step_created` ahead of it). They have to count: the client takes + * `stateUpdatedAt` over a log that includes what it just appended, so + * counting less would leave the count below the watermark it is paired with + * and reject perfectly current writes. + * + * A scan that starts without a cursor replaces the set rather than adding to + * it β€” that is a fresh delivery re-reading the log from the beginning, and its + * earlier view should not linger. + */ + const loadedEvents = new Map>(); + + function loadedSet(runId: string): Set { + let set = loadedEvents.get(runId); + if (!set) { + set = new Set(); + loadedEvents.set(runId, set); + } + return set; + } + + function noteLoadedEvents( + runId: string, + args: readonly unknown[], + result: unknown + ): void { + const page = (result as { data?: { eventId?: string }[] } | undefined) + ?.data; + if (!page) return; + const cursor = (args[0] as { pagination?: { cursor?: string } } | undefined) + ?.pagination?.cursor; + if (!cursor) loadedEvents.set(runId, new Set()); + const set = loadedSet(runId); + for (const event of page) if (event.eventId) set.add(event.eventId); + } + + /** The `stateEventCount` the loaded log implies at `stateUpdatedAt`. */ + function loadedCount( + runId: string | undefined, + stateUpdatedAt: number + ): number { + if (!runId) return 0; + let count = 0; + for (const eventId of loadedSet(runId)) { + if (ulidTimeOf(eventId) <= stateUpdatedAt) count++; + } + return count; + } + + /** Wrap one world method so it becomes a call point. */ + function intercept(call: WorldCallName, fn: F): F { + return (async (...args: Parameters) => { + const runId = runIdOf(call, args); + const request = + call === 'events.create' + ? (args[1] as CallContext['request']) + : undefined; + const writer = writerOfCall(call, runId, request); + + const base: CallContext = { + seq: callSeq++, + call, + phase: 'before', + writer, + args, + atMs: clock.now(), + runId, + ...(request ? { request } : {}), + ...(call === 'queue' ? { message: args[1] as QueuePayload } : {}), + }; + + record(base); + await fireWatches(base); + + // The handler boundary. workflow-server mints the event id here + // (`EventId.make()`, before the storage write is attempted) because + // DynamoDB does not generate ids and that id *is* the log's sort key. So a + // write acquires its position and its visibility at two different moments. + // A scenario holds that gap open with `sim.beginHookDelivery`, which takes + // the position on one side and lets the write land on the other. + let callArgs = args; + let entered = base; + if (call === 'events.create') { + // A reserved position wins: it belongs to a write whose handler was + // entered earlier and is only now reaching storage. + const minted = reservedPosition ?? store.mintEvent(); + reservedPosition = undefined; + const params = (args[2] ?? {}) as Record; + callArgs = [ + args[0], + args[1], + { + ...params, + minted, + // The runtime's own count wins when it sent one. Since #3145 + // `preconditionSnapshotParams` puts `stateEventCount` on every + // replay-context create, so the value under test is normally the + // real client's, not ours. + // + // The reconstruction is the fallback, for the writes core does not + // count: a step body committing outside a replay context, and any + // create made while the precondition guard's env kill-switch is off + // (`preconditionSnapshotParams` returns `{}` wholesale then). It is + // the size of the last page this writer read, which is what the + // client would have counted had it counted. + ...(options.countGuard && + !isExternal() && + typeof params.stateUpdatedAt === 'number' + ? { + stateEventCount: + typeof params.stateEventCount === 'number' + ? params.stateEventCount + : loadedCount(runId, params.stateUpdatedAt), + } + : {}), + }, + ] as unknown as Parameters; + entered = { ...base, args: callArgs }; + } + + // For a create, what the log held going in β€” so the caller can be + // credited with everything its own write appended, not just the event the + // call handed back. A `step_started` claim also appends the `step_created` + // ahead of it, and a client that did not count both would look like it was + // holding a hole it had itself just made. + const before = + call === 'events.create' && runId && !isExternal() + ? new Set(store.allEvents(runId).map((e) => e.eventId)) + : undefined; + + let result: unknown; + let error: unknown; + let threw = false; + try { + result = await fn(...callArgs); + } catch (err) { + error = err; + threw = true; + } + + if (!threw && runId && !isExternal()) { + if (call === 'events.list') { + noteLoadedEvents(runId, args, result); + } else if (before) { + const set = loadedSet(runId); + for (const event of store.allEvents(runId)) { + if (!before.has(event.eventId)) set.add(event.eventId); + } + } + } + + const after: CallContext = { + ...entered, + phase: 'after', + atMs: clock.now(), + ...(threw ? { error } : {}), + ...(call === 'events.create' && !threw + ? { event: (result as { event?: CallContext['event'] })?.event } + : {}), + }; + + if (threw) { + // Recorded unconditionally. A rejected write is how a run self-corrects + // under the optimistic-concurrency fence, so it belongs in the trace by + // default rather than only when a scenario thought to look for it. + const rejection: RejectedCall = { + seq: base.seq, + call, + writer: base.writer, + errorName: error instanceof Error ? error.name : typeof error, + message: error instanceof Error ? error.message : String(error), + ...(request?.eventType ? { eventType: request.eventType } : {}), + }; + rejected.push(rejection); + pushTrace({ + kind: 'warn', + message: + `world rejected ${call}` + + `${rejection.eventType ? ` ${rejection.eventType}` : ''}` + + ` from ${base.writer}: ${rejection.errorName}: ${rejection.message}`, + }); + } + + record(after); + await fireWatches(after); + + if (threw) throw error; + return result; + }) as F; + } + + /** Wrap a namespace of world methods, keyed by their call-point name. */ + function interceptAll>( + target: T, + names: { [K in keyof T]?: WorldCallName } + ): T { + const out: Record = {}; + for (const [key, call] of Object.entries(names) as [ + string, + WorldCallName, + ][]) { + const fn = target[key]; + if (typeof fn !== 'function') continue; + out[key] = intercept(call, (fn as AsyncFn).bind(target)); + } + return out as T; + } + + const world: SimWorld = { + specVersion: SPEC_VERSION_CURRENT, + capabilities: { + // Only advertise the fence when the store is actually enforcing it β€” a + // runtime fast path gated on this capability must never run without one. + ...(options.preconditionGuard ? { preconditionGuard: true } : {}), + }, + getDeploymentId: intercept('getDeploymentId', () => + simQueue.getDeploymentId() + ), + queue: intercept('queue', simQueue.queue), + createQueueHandler: simQueue.createQueueHandler, + createRunId: () => ids.ulid(), + + runs: interceptAll(store.runs, { + get: 'runs.get', + list: 'runs.list', + }), + steps: interceptAll(store.steps, { + get: 'steps.get', + list: 'steps.list', + }), + events: interceptAll(store.events, { + create: 'events.create', + get: 'events.get', + list: 'events.list', + listByCorrelationId: 'events.listByCorrelationId', + }), + hooks: interceptAll(store.hooks, { + get: 'hooks.get', + getByToken: 'hooks.getByToken', + list: 'hooks.list', + }), + streams: interceptAll(streamer.streams, { + write: 'streams.write', + writeMulti: 'streams.writeMulti', + close: 'streams.close', + get: 'streams.get', + list: 'streams.list', + getChunks: 'streams.getChunks', + getInfo: 'streams.getInfo', + }), + + clock, + ids, + store, + simQueue, + streamer, + snapshot, + trace, + + registerHandler: (prefix, handler) => + simQueue.registerHandler(prefix as never, handler), + addWatch(watch) { + const entry = { watch, matches: 0 }; + watches.push(entry); + return () => { + const at = watches.indexOf(entry); + if (at !== -1) watches.splice(at, 1); + }; + }, + watchErrors: () => [...watchErrors], + setScenarioApi(resolve) { + resolveApi = resolve; + }, + async asExternal(fn) { + return externalCtx.run(true, fn); + }, + reservePosition: () => store.mintEvent(), + async withReservedPosition(position, fn) { + reservedPosition = position; + try { + return await fn(); + } finally { + reservedPosition = undefined; + } + }, + pushTrace, + callCount: () => callSeq, + callHistory: () => history, + rejections: () => rejected, + }; + + return world; +} diff --git a/packages/world-sim/src/writers.test.ts b/packages/world-sim/src/writers.test.ts new file mode 100644 index 0000000000..f3ec346a06 --- /dev/null +++ b/packages/world-sim/src/writers.test.ts @@ -0,0 +1,164 @@ +/** + * The writer layer's guarantees, pinned against the inert handler. + * + * Three of them matter enough to test directly, because each one is a way the + * termination guarantee could be lost: + * + * - a point that has already gone by must be an error, not a wait; + * - a point that will never arrive must time out with a diagnosis naming the + * writer, not consume the scenario's whole wall-clock budget; + * - re-advancing a writer must release the call it was holding, or the second + * `runTo` waits behind a call that nobody let go of. + */ + +import { describe, expect, it } from 'vitest'; +import { runScenario, type ScenarioSpec } from './scenario.js'; + +/** A flow handler that accepts every delivery and advances nothing. */ +const inertHandler = async () => Response.json({ ok: true }); + +const WORKFLOW_ID = 'workflow//./workflows/demo//demoWorkflow'; + +function scenario(partial: Partial): ScenarioSpec { + return { + name: 'test', + workflow: { workflowId: WORKFLOW_ID }, + limits: { maxWallMs: 1_000 }, + ...partial, + }; +} + +describe('writer runTo', () => { + it('holds the call that produced the event, before it is committed', async () => { + let eventsWhileHeld: number | undefined; + await runScenario( + scenario({ + script: async (sim) => { + const wf = sim.writer.orchestrator(); + const held = await wf.runToEventProduced('run_created'); + // `produced` is the `before` phase: the write has been decided but + // the log does not have it yet. + eventsWhileHeld = sim.world.events().length; + await held.release(); + }, + }), + { handler: inertHandler } + ); + expect(eventsWhileHeld).toBe(0); + }); + + it('holds after the commit for `committed`', async () => { + let seen: string[] = []; + await runScenario( + scenario({ + script: async (sim) => { + const wf = sim.writer.orchestrator(); + await wf.runToEventCommitted('run_created'); + seen = sim.world + .events() + .map((e: { eventType: string }) => e.eventType); + await wf.release(); + }, + }), + { handler: inertHandler } + ); + expect(seen).toEqual(['run_created']); + }); + + it('reports a point the writer sailed past instead of waiting for it', async () => { + const result = await runScenario( + scenario({ + script: async (sim) => { + const wf = sim.writer.orchestrator(); + // Stop before the commit, then let go: the commit happens while the + // script is not watching for it, so asking for it now is the + // arm-too-late mistake, and it must be reported as one. + const held = await wf.runToEventProduced('run_created'); + await held.release(); + await wf.runToEventCommitted('run_created'); + }, + }), + { handler: inertHandler } + ); + + expect(result.ok).toBe(false); + const text = result.problems.join('\n'); + // The point is named, so the failure says what happened rather than only + // that something did not. + expect(text).toMatch(/already passed run_created \(committed\)/); + // And it explains the fix, since the fix is not obvious. + expect(text).toMatch(/level-triggered/); + }); + + it('times out one runTo without spending the scenario budget', async () => { + const result = await runScenario( + scenario({ + limits: { maxWallMs: 5_000, maxRunToWallMs: 200 }, + script: async (sim) => { + const wf = sim.writer.orchestrator(); + // Hold the scheduler so the scenario cannot end on its own, then wait + // for something the inert handler will never do. + const held = await wf.runToEventProduced('run_created'); + try { + await sim.writer + .step('never') + .runToEventCommitted('step_completed'); + } finally { + await held.release(); + } + }, + }), + { handler: inertHandler } + ); + + expect(result.ok).toBe(false); + const text = result.problems.join('\n'); + expect(text).toMatch(/step:never did not reach step_completed/); + // The report says where every writer was standing, which is the whole + // reason this budget exists separately from the global one. + expect(text).toMatch(/orchestrator HELD at/); + // The specific diagnosis won, and the global deadline never fired. + expect(text).not.toMatch(/wall-clock budget/); + expect(result.wallMs).toBeLessThan(4_000); + }); + + it('releases the previous hold when the writer is advanced again', async () => { + let events: string[] = []; + const result = await runScenario( + scenario({ + script: async (sim) => { + const wf = sim.writer.orchestrator(); + await wf.runToEventProduced('run_created'); + // No explicit release: the next advance must let the call finish, + // otherwise this second wait blocks behind a call nobody let go of. + await wf.runToEventCommitted('run_created'); + events = sim.world + .events() + .map((e: { eventType: string }) => e.eventType); + await wf.release(); + }, + }), + { handler: inertHandler } + ); + + expect(events).toEqual(['run_created']); + expect(result.problems.filter((p) => p.includes('script'))).toEqual([]); + expect(result.outcome).toBe('stalled'); // the inert run makes no progress + }); + + it('names the writers it has seen', async () => { + let seen: string[] = []; + await runScenario( + scenario({ + script: async (sim) => { + const wf = sim.writer.orchestrator(); + await wf.runToEventCommitted('run_created'); + seen = sim.writer.seen(); + await wf.release(); + }, + }), + { handler: inertHandler } + ); + expect(seen).toContain('orchestrator'); + }); +}); diff --git a/packages/world-sim/src/writers.ts b/packages/world-sim/src/writers.ts new file mode 100644 index 0000000000..971f7e5a75 --- /dev/null +++ b/packages/world-sim/src/writers.ts @@ -0,0 +1,422 @@ +/** + * Writers: the scenario's unit of scheduling. + * + * The property under test is that a workflow run has **several concurrent + * writers appending to one event log with no serializable isolation between + * them**. The orchestrator loads the log, replays the workflow against it, and + * commits what that decides; each step body writes its own result; a webhook + * receiver writes a `hook_received` whenever it likes. Nothing sequences those + * against each other, and the World API exposes no primitive that could β€” at + * most an optimistic fence, which is two checks and not isolation: + * + * - `stateUpdatedAt` is a high-water mark on one class of write ("is there an + * out-of-band event newer than my snapshot?"). It sees a log truncated at the + * end; it cannot see a hole in the middle. + * - a count of the events the caller loaded at or below that mark closes the + * hole, but only for events already committed when the write is checked, and + * only within a bounded window of the log's tail. It is also dark in + * production, since no client sends the count (`countGuard` arms it). + * + * So a hole in the middle is what the step-vs-step scenarios exploit, and a hole + * that opens *after* the write it should have fenced is beyond either check. + * + * A real deployment resolves that by racing. This module resolves it by + * *naming* the writers and letting a scenario advance them one at a time, so + * the interleaving is a statement in the script rather than an accident: + * + * ```ts + * const wf = sim.writer.orchestrator(); + * const slow = sim.writer.step('slow'); + * + * await wf.runToEventCommitted('step_started', 'reserveInventory'); + * await sim.deliverHook('approval:doc-1', { approved: true }); + * await wf.release(); + * ``` + * + * The hook is committed after `step_started` is durable and before the workflow + * resumes, because the orchestrator is *stopped inside the call that committed + * it*. That is a fact about the run, not a race the scenario won. + * + * ## Level-triggered, deliberately + * + * `runTo` consults the history of points each writer has already reached before + * arming anything. If the point has gone by it throws, naming the call it + * happened at. The alternative β€” arm a watch and wait β€” is a hang: a held call + * blocks its writer, and in the limit blocks the scheduler, so there is no + * quiescence to fall back on and no timer to eventually fire. An edge-triggered + * wait on an edge that has passed is the one way to lose the termination + * guarantee this package is built around, so it is made impossible rather than + * merely documented. + * + * ## What is not offered + * + * Writers form a dependency graph β€” the orchestrator awaits its own step + * bodies β€” so not every interleaving exists to be asked for, and an + * unsatisfiable `runTo` can only be reported, not prevented. Each wait + * therefore carries its own wall-clock budget and, on expiry, reports where + * every writer was standing. + * + * Concurrent *invocations* are also out of reach: the scheduler delivers one + * queue message at a time, so `orchestrator` names one delivery's worth of + * orchestration. Two step bodies inside one delivery are genuinely concurrent + * and separately steerable, which is enough to reach the interesting corruption + * without a second invocation. + */ + +import type { + CallMatch, + CallPhase, + Held, + ObservedPoint, + RunToOptions, + WorldCallName, + Writer, + WriterHandles, + WriterId, +} from './types.js'; +import type { SimWorld } from './world.js'; + +/** + * Thrown when a `runTo` names a point its writer has already gone past. Not a + * simulator failure β€” a scenario that armed too late. + */ +export class AlreadyPassedError extends Error { + override readonly name = 'AlreadyPassedError'; +} + +/** Thrown when a `runTo` blows its own wall-clock budget. */ +export class RunToTimeoutError extends Error { + override readonly name = 'RunToTimeoutError'; +} + +/** A hold armed but not yet reached. `cancel` disposes it and rejects the wait. */ +export interface ArmedHold { + reached: Promise<{ ctx: Held['ctx']; release(): void }>; + cancel(reason: Error): void; +} + +/** How a hold gets armed. Supplied by the tempo layer. */ +export type Arm = (match: CallMatch, label: string) => ArmedHold; + +export interface WriterRegistry { + handles: WriterHandles; + /** Where every writer is standing right now, for an error report. */ + describe(): string; + /** Forget every local hold, because the scenario released them wholesale. */ + forgetHolds(): void; +} + +/** The fields that identify a point, shared by the watch and the level check. */ +interface PointSpec { + phase: CallPhase; + eventTypes?: string[]; + calls?: WorldCallName[]; + stepName?: string; + token?: string; + correlationId?: string; + /** Mirrors `CallMatch.failed`; `undefined` accepts either outcome. */ + failed?: boolean; +} + +function asOptions(options: string | RunToOptions | undefined): RunToOptions { + return typeof options === 'string' ? { stepName: options } : (options ?? {}); +} + +function toArray(value: T | T[] | undefined): T[] | undefined { + if (value === undefined) return undefined; + return Array.isArray(value) ? value : [value]; +} + +function describePoint(spec: PointSpec): string { + const bits = [ + ...(spec.eventTypes ?? []), + ...(spec.calls ?? []), + spec.stepName ? `step=${spec.stepName}` : '', + spec.token ? `token=${spec.token}` : '', + spec.correlationId ? `correlation=${spec.correlationId}` : '', + ].filter(Boolean); + const where = spec.phase === 'before' ? 'produced' : 'committed'; + return `${bits.join(' ') || 'any call'} (${where})`; +} + +/** Does a remembered point satisfy the spec? The level-triggered half. */ +function pointMatches(spec: PointSpec, point: ObservedPoint): boolean { + if (point.phase !== spec.phase) return false; + if (spec.eventTypes) { + if (point.call !== 'events.create') return false; + if (!point.eventType || !spec.eventTypes.includes(point.eventType)) { + return false; + } + } else if (spec.calls && !spec.calls.includes(point.call)) { + return false; + } + if (spec.stepName && point.stepName !== spec.stepName) return false; + if (spec.token && point.token !== spec.token) return false; + if (spec.correlationId && point.correlationId !== spec.correlationId) { + return false; + } + if (spec.failed !== undefined && spec.failed !== point.failed) return false; + return true; +} + +export function createWriters(deps: { + world: SimWorld; + arm: Arm; + defaultTimeoutMs: number; +}): WriterRegistry { + const { world, arm } = deps; + const created: { + label: string; + heldAt(): string | undefined; + forget(): void; + }[] = []; + + /** The last few points a writer reached, for an error report. */ + function recentPoints(pred: (writer: WriterId) => boolean): string { + const points = world + .callHistory() + .filter((p) => pred(p.writer)) + .slice(-4) + .map( + (p) => + `#${p.seq} ${p.call}:${p.phase}${p.eventType ? ` ${p.eventType}` : ''}${ + p.stepName ? `/${p.stepName}` : '' + }` + ); + return points.length > 0 ? points.join(', ') : 'nothing yet'; + } + + /** + * Where in the recorded history a reached hold sits. + * + * The watch hands over a `CallContext`, and a call is recorded once per phase + * under the same `seq`, so the pair identifies the record. Searching from the + * end finds it immediately in the normal case. A history that hit its cap has + * stopped recording, so fall back to "everything recorded is behind us" β€” + * over-consuming is safe, under-consuming would report a spurious miss. + */ + function ordinalReached(ctx: Held['ctx']): number { + const history = world.callHistory(); + for (let i = history.length - 1; i >= 0; i--) { + const point = history[i]; + if (point.seq === ctx.seq && point.phase === ctx.phase) { + return point.ordinal; + } + } + return history.length > 0 ? history[history.length - 1].ordinal : -1; + } + + function describe(): string { + const parts = created.map((w) => { + const at = w.heldAt(); + return at ? `${w.label} HELD at ${at}` : `${w.label} running`; + }); + const seen = handles.seen(); + parts.push(`writers seen: ${seen.join(', ') || 'none'}`); + return parts.join('; '); + } + + function makeWriter( + id: WriterId, + label: string, + pred: (writer: WriterId) => boolean + ): Writer { + /** + * How far this writer has been advanced, as a history ordinal. Points at or + * before it are "already consumed" and do not count as already-passed: + * asking twice for `step_completed` means the *next* one, which is what the + * duplicate-delivery scenarios need. A consumed point with no successor is + * therefore not an error here β€” it is the per-`runTo` timeout's business. + */ + let watermark = -1; + let hold: { ctx: Held['ctx']; release(): void; done: boolean } | undefined; + + const release = async (): Promise => { + if (!hold || hold.done) return; + hold.done = true; + const target = hold; + hold = undefined; + target.release(); + // Yield a full turn so that once this resolves the released writer has + // actually moved. Without it `await release()` returns while the call is + // still sitting in the microtask queue, and a scenario that reads the log + // straight afterwards sees the state it was trying to leave behind. + await new Promise((resolve) => { + setImmediate(resolve); + }); + }; + + async function runTo( + spec: PointSpec, + options: RunToOptions + ): Promise { + const what = describePoint(spec); + const name = options.label ?? `${label} -> ${what}`; + + // The level-triggered check. `where` is a predicate on world state at the + // moment of the point, and world state has moved on since, so it cannot + // be re-evaluated against history: with a `where` this degrades to + // edge-triggered and leans on the timeout instead. + if (!options.where) { + const passed = world + .callHistory() + .find( + (p) => + p.ordinal > watermark && pred(p.writer) && pointMatches(spec, p) + ); + if (passed) { + throw new AlreadyPassedError( + `${label} has already passed ${what} β€” it happened at call #${passed.seq}` + + (passed.depth > 0 + ? `, inside another call the scenario was holding, where a hold is not possible` + : '') + + `.\n` + + `runTo is level-triggered: it reports a missed point instead of arming a hold ` + + `that can never fire, because a hold that never fires is a hang.\n` + + `Arm the hold before the point is reached β€” start every wait, then await them:\n` + + ` const a = first.runToEventProduced('step_completed');\n` + + ` const b = second.runToEventProduced('step_completed');\n` + + ` await a; await b;\n` + + `State: ${describe()}` + ); + } + } + + const match: CallMatch = { + phase: spec.phase, + writer: pred, + ...(spec.eventTypes ? { eventType: spec.eventTypes as never } : {}), + ...(spec.calls ? { call: spec.calls } : {}), + ...(spec.stepName ? { stepName: spec.stepName } : {}), + ...(spec.token ? { token: spec.token } : {}), + ...(spec.correlationId ? { correlationId: spec.correlationId } : {}), + ...(spec.failed !== undefined ? { failed: spec.failed } : {}), + ...(options.where + ? { where: (_ctx, snapshot) => options.where?.(snapshot) ?? true } + : {}), + }; + + const armed = arm(match, name); + const budget = options.timeoutMs ?? deps.defaultTimeoutMs; + const timer = setTimeout(() => { + armed.cancel( + new RunToTimeoutError( + `${label} did not reach ${what} within ${budget}ms.\n` + + `State: ${describe()}\n` + + `${label} has reached: ${recentPoints(pred)}` + ) + ); + }, budget); + + try { + // Advancing a writer that is already held means "let it go, then stop + // it at the next thing" β€” the reading `runTo` after `runTo` invites. + // The release happens *after* the watch is armed, and that order is + // load-bearing: the released writer can reach the next point within the + // same turn β€” the `after` phase of the very call it was held in is the + // common case β€” and a watch armed afterwards would have missed it. + await release(); + const reached = await armed.reached; + watermark = ordinalReached(reached.ctx); + hold = { ctx: reached.ctx, release: reached.release, done: false }; + return { + writer: reached.ctx.writer, + ctx: reached.ctx, + release, + }; + } finally { + clearTimeout(timer); + } + } + + const writer: Writer = { + id, + runToEventProduced: (eventType, options) => { + const opts = asOptions(options); + return runTo( + { + phase: 'before', + eventTypes: toArray(eventType), + ...(opts.stepName ? { stepName: opts.stepName } : {}), + ...(opts.token ? { token: opts.token } : {}), + ...(opts.correlationId + ? { correlationId: opts.correlationId } + : {}), + }, + opts + ); + }, + runToEventCommitted: (eventType, options) => { + const opts = asOptions(options); + return runTo( + { + phase: 'after', + // "Committed" means committed. Without this a rejected create + // matches too, and under the fence a `PreconditionFailedError` is + // routine β€” the script would resume believing a write is durable + // when it 412'd, and the watermark would consume the point, so the + // retry's real commit would read as the *next* one. + failed: false, + eventTypes: toArray(eventType), + ...(opts.stepName ? { stepName: opts.stepName } : {}), + ...(opts.token ? { token: opts.token } : {}), + ...(opts.correlationId + ? { correlationId: opts.correlationId } + : {}), + }, + opts + ); + }, + release, + isHeld: () => hold !== undefined && !hold.done, + history: () => world.callHistory().filter((p) => pred(p.writer)), + }; + + created.push({ + label, + heldAt: () => + hold && !hold.done + ? `${hold.ctx.call}:${hold.ctx.phase}${ + hold.ctx.request?.eventType + ? ` ${hold.ctx.request.eventType}` + : '' + }` + : undefined, + forget: () => { + if (hold) hold.done = true; + hold = undefined; + }, + }); + + return writer; + } + + const handles: WriterHandles = { + orchestrator: () => + makeWriter('orchestrator', 'orchestrator', (w) => w === 'orchestrator'), + step: (shortName) => + makeWriter( + `step:${shortName}`, + `step:${shortName}`, + (w) => w === `step:${shortName}` + ), + anyStep: () => + makeWriter('step:*', 'any step body', (w) => w.startsWith('step:')), + any: () => makeWriter('*', 'any writer', () => true), + seen: () => { + const out: WriterId[] = []; + for (const point of world.callHistory()) { + if (!out.includes(point.writer)) out.push(point.writer); + } + return out; + }, + }; + + return { + handles, + describe, + forgetHolds: () => { + for (const w of created) w.forget(); + }, + }; +} diff --git a/packages/world-sim/tsconfig.json b/packages/world-sim/tsconfig.json new file mode 100644 index 0000000000..e40ca52d01 --- /dev/null +++ b/packages/world-sim/tsconfig.json @@ -0,0 +1,10 @@ +{ + "extends": "@workflow/tsconfig/base.json", + "compilerOptions": { + "outDir": "dist", + "rootDir": "./src", + "target": "es2022" + }, + "include": ["src"], + "exclude": ["node_modules", "**/*.test.ts"] +} diff --git a/packages/world-vercel/src/events-v4.test.ts b/packages/world-vercel/src/events-v4.test.ts index 23ec038eab..161195fe26 100644 --- a/packages/world-vercel/src/events-v4.test.ts +++ b/packages/world-vercel/src/events-v4.test.ts @@ -143,6 +143,116 @@ describe('throwForErrorResponse', () => { /createEvent failed: HTTP 500 plain text oops/ ); }); + + it('reads message and code out of a CBOR body', () => { + // A 412 that carries an event delta answers in CBOR so the delta's + // payloads stay real bytes. Decoding it by content-type is what keeps the + // message and code from being lost to a failed JSON.parse. + try { + throwForErrorResponse( + 412, + { 'content-type': 'application/cbor' }, + encode({ message: 'Event log moved on' }), + 'createEvent', + 'http://x' + ); + expect.unreachable(); + } catch (err) { + expect(PreconditionFailedError.is(err)).toBe(true); + expect((err as PreconditionFailedError).message).toBe( + 'Event log moved on' + ); + } + + try { + throwForErrorResponse( + 404, + { 'content-type': 'application/cbor' }, + encode({ message: 'hook not found', code: 'not_found' }), + 'createEvent', + 'http://x' + ); + expect.unreachable(); + } catch (err) { + expect((err as WorkflowWorldError).code).toBe('not_found'); + } + }); + + it('keeps a CBOR 412 delta whose event payload is real bytes', () => { + const result = new TextEncoder().encode('"done"'); + try { + throwForErrorResponse( + 412, + { 'content-type': 'application/cbor' }, + encode({ + message: 'Event log moved on', + cursor: 'eid:evnt_missing', + events: [ + { + eventId: 'evnt_missing', + runId: 'wrun_1', + eventType: 'step_completed', + correlationId: 'step_0', + specVersion: 5, + createdAt: '2026-06-10T00:00:00.000Z', + eventData: { result }, + }, + ], + }), + 'createEvent', + 'http://x' + ); + expect.unreachable(); + } catch (err) { + const details = (err as PreconditionFailedError).details as { + events: Array<{ eventData: { result: unknown } }>; + cursor?: string; + }; + expect(details.cursor).toBe('eid:evnt_missing'); + // A JSON body would have mangled these bytes and the delta would have + // been refused whole; CBOR round-trips them, so the client can merge. + expect(details.events[0]?.eventData.result).toBeInstanceOf(Uint8Array); + expect( + new TextDecoder().decode( + details.events[0]?.eventData.result as Uint8Array + ) + ).toBe('"done"'); + } + }); + + it('falls back to the default message when a CBOR body will not decode', () => { + // Undecodable bytes must not be appended to the message as mojibake. + const garbage = new Uint8Array([0xff, 0xfe, 0xfd]); + try { + throwForErrorResponse( + 500, + { 'content-type': 'application/cbor' }, + garbage, + 'createEvent', + 'http://x' + ); + expect.unreachable(); + } catch (err) { + expect((err as WorkflowWorldError).message).toBe( + 'v4 createEvent failed: HTTP 500' + ); + } + }); + + it('still parses a JSON body delivered as bytes', () => { + try { + throwForErrorResponse( + 404, + { 'content-type': 'application/json' }, + new TextEncoder().encode('{"message":"hook not found"}'), + 'createEvent', + 'http://x' + ); + expect.unreachable(); + } catch (err) { + expect((err as WorkflowWorldError).message).toBe('hook not found'); + } + }); }); /** @@ -1230,6 +1340,119 @@ describe('createWorkflowRunEventV4 over HTTP', () => { agent.assertNoPendingInterceptors(); }); + it('forwards maxSlot in the frame meta', async () => { + const origin = + WORKFLOW_SERVER_URL_OVERRIDE || 'https://vercel-workflow.com'; + const agent = new MockAgent(); + agent.disableNetConnect(); + + let capturedMeta: Record | undefined; + agent + .get(origin) + .intercept({ + path: '/api/v4/runs/wrun_1/events/wait_created', + method: 'POST', + }) + .reply( + 200, + (opts: { body?: unknown }) => { + const bytes = new Uint8Array(opts.body as ArrayBufferLike); + const metaLen = new DataView( + bytes.buffer, + bytes.byteOffset, + bytes.byteLength + ).getUint32(0, false); + capturedMeta = decode(bytes.subarray(4, 4 + metaLen)) as Record< + string, + unknown + >; + return createEventBody({ + eventType: 'wait_created', + specVersion: 5, + correlationId: 'wait_1', + eventData: { resumeAt: CREATED_AT }, + }); + }, + { + headers: { + 'x-wf-event-id': 'evnt_1', + 'x-wf-run-id': 'wrun_1', + 'x-wf-created-at': '2026-06-10T00:00:00.000Z', + }, + } + ); + + await createWorkflowRunEventV4( + { + runId: 'wrun_1', + eventType: 'wait_created', + specVersion: 6, + correlationId: 'wait_1', + maxSlot: 12, + }, + { token: 'test-token', dispatcher: agent } + ); + + expect(capturedMeta?.maxSlot).toBe(12); + agent.assertNoPendingInterceptors(); + }); + + it('omits maxSlot from the frame meta when not set', async () => { + const origin = + WORKFLOW_SERVER_URL_OVERRIDE || 'https://vercel-workflow.com'; + const agent = new MockAgent(); + agent.disableNetConnect(); + + let capturedMeta: Record | undefined; + agent + .get(origin) + .intercept({ + path: '/api/v4/runs/wrun_1/events/wait_created', + method: 'POST', + }) + .reply( + 200, + (opts: { body?: unknown }) => { + const bytes = new Uint8Array(opts.body as ArrayBufferLike); + const metaLen = new DataView( + bytes.buffer, + bytes.byteOffset, + bytes.byteLength + ).getUint32(0, false); + capturedMeta = decode(bytes.subarray(4, 4 + metaLen)) as Record< + string, + unknown + >; + return createEventBody({ + eventType: 'wait_created', + specVersion: 5, + correlationId: 'wait_1', + eventData: { resumeAt: CREATED_AT }, + }); + }, + { + headers: { + 'x-wf-event-id': 'evnt_1', + 'x-wf-run-id': 'wrun_1', + 'x-wf-created-at': '2026-06-10T00:00:00.000Z', + }, + } + ); + + await createWorkflowRunEventV4( + { + runId: 'wrun_1', + eventType: 'wait_created', + specVersion: 5, + correlationId: 'wait_1', + }, + { token: 'test-token', dispatcher: agent } + ); + + expect('maxSlot' in (capturedMeta ?? {})).toBe(false); + agent.assertNoPendingInterceptors(); + }); + it('omits stateEventCount and stateCursor from the frame meta when not set', async () => { const origin = WORKFLOW_SERVER_URL_OVERRIDE || 'https://vercel-workflow.com'; diff --git a/packages/world-vercel/src/events-v4.ts b/packages/world-vercel/src/events-v4.ts index ab712abf95..f68f35c880 100644 --- a/packages/world-vercel/src/events-v4.ts +++ b/packages/world-vercel/src/events-v4.ts @@ -104,11 +104,13 @@ async function fetchV4( noteEventsTransportOutcome(dispatcher, error), timeoutMs: null, logLabel: opName, + // Read the body as bytes, not text: a CBOR error body (the fence 412 + // carries event payloads back) does not survive a UTF-8 decode. buildError: async (response) => errorFromV4Response( response.status, headersToRecord(response.headers), - await response.text(), + new Uint8Array(await response.arrayBuffer()), opName, url ), @@ -242,6 +244,19 @@ interface CreateEventV4InputBase { * on for the *accepted* path. */ stateCursor?: string; + /** + * Highest event slot the writer had loaded, i.e. the length of its loaded + * log under slot identity. Named `maxSlot` on the wire because the meta + * already carries an unrelated telemetry `eventCount`. + * + * Supersedes the `stateUpdatedAt`/`stateEventCount`/`stateCursor` triple for + * slot-identity runs: with dense positions one integer says everything the + * watermark approximated. The server allocates from the tail regardless, and + * uses this only to report which slots the write skipped over (returned on + * the success response as `events`/`cursor`/`hasMore`). Older servers ignore + * it. + */ + maxSlot?: number; /** Number of consecutive replay divergences resolved by this write. */ replayDivergenceCount?: number; /** Content digest of the serialized resume payload. Forwarded alongside @@ -448,6 +463,7 @@ function buildPostFrameMeta( meta.stateEventCount = input.stateEventCount; } if (input.stateCursor !== undefined) meta.stateCursor = input.stateCursor; + if (input.maxSlot !== undefined) meta.maxSlot = input.maxSlot; if (input.replayDivergenceCount !== undefined) { meta.replayDivergenceCount = input.replayDivergenceCount; } @@ -469,26 +485,25 @@ function buildPostFrameMeta( function errorFromV4Response( statusCode: number, responseHeaders: Record, - errorBody: string, + errorBody: string | Uint8Array, opName: string, url: string ): Error { let message = `v4 ${opName} failed: HTTP ${statusCode}`; let code: string | undefined; let details: unknown; - try { - const json = JSON.parse(errorBody) as { - message?: string; - code?: string; - events?: unknown; - cursor?: unknown; - }; - if (typeof json.message === 'string') message = json.message; - if (typeof json.code === 'string') code = json.code; - if (statusCode === 412) details = decodePreconditionDetails(json); - } catch { - // body wasn't JSON β€” keep the default message, append raw text below - if (errorBody) message += ` ${errorBody}`; + const { record, text } = parseV4ErrorBody( + errorBody, + readHeader(responseHeaders, 'content-type') + ); + if (record) { + if (typeof record.message === 'string') message = record.message; + if (typeof record.code === 'string') code = record.code; + if (statusCode === 412) details = decodePreconditionDetails(record); + } else if (text) { + // body wasn't a structured object β€” keep the default message and append + // whatever the server did send + message += ` ${text}`; } const retryAfter = parseRetryAfter( @@ -505,6 +520,55 @@ function errorFromV4Response( }); } +/** The fields `errorFromV4Response` reads off a structured error body. */ +interface V4ErrorBody { + message?: unknown; + code?: unknown; + events?: unknown; + cursor?: unknown; +} + +/** + * Decode an error body into the record the error builder reads, or into the + * raw text to append when it is not structured. + * + * Two encodings reach this. The default is JSON: the v4 request sends no + * `Accept: application/cbor`, so the server's generic error responder + * negotiates JSON. Responses that need to carry event payloads back are + * hand-encoded as CBOR by the server and say so in `content-type`, because + * JSON cannot round-trip a `Uint8Array` (see `hasUnusablePayload`). Reading + * the body as bytes and branching on the header serves both; decoding bytes as + * text first would corrupt CBOR beyond recovery. + */ +function parseV4ErrorBody( + body: string | Uint8Array, + contentType: string | undefined +): { record?: V4ErrorBody; text?: string } { + if (typeof body !== 'string' && contentType?.includes('application/cbor')) { + try { + // cbor-x caches decode state on its input; decode a copy so a shared + // buffer is never mutated under an unrelated reader. + const decoded = decode(body.slice()) as unknown; + if (typeof decoded === 'object' && decoded !== null) { + return { record: decoded as V4ErrorBody }; + } + } catch { + // undecodable CBOR: appending its bytes as text would be noise + } + return {}; + } + const text = typeof body === 'string' ? body : new TextDecoder().decode(body); + try { + const json = JSON.parse(text) as unknown; + if (typeof json === 'object' && json !== null) { + return { record: json as V4ErrorBody }; + } + } catch { + // not JSON either β€” fall through to the raw text + } + return { text }; +} + /** * Pick the inline event delta off a 412 body. * @@ -520,10 +584,9 @@ function errorFromV4Response( * untrusted-shaped data on a failure path, and the fallback (a full reload) is * always correct. */ -function decodePreconditionDetails(json: { - events?: unknown; - cursor?: unknown; -}): PreconditionFailureDetails | undefined { +function decodePreconditionDetails( + json: V4ErrorBody +): PreconditionFailureDetails | undefined { if (!Array.isArray(json.events) || json.events.length === 0) return undefined; const events: Event[] = []; for (const raw of json.events) { @@ -548,17 +611,19 @@ function decodePreconditionDetails(json: { * Payload fields (input / output / result / error / payload / metadata) are * `Uint8Array` everywhere else in this client β€” the runtime dehydrates before * writing and rehydrates after reading, and the write path throws on anything - * else. A 412 body is JSON, though: the request carries no - * `Accept: application/cbor`, so resolved bytes serialize to + * else. A JSON 412 body cannot hold that: resolved bytes serialize to * `{"type":"Buffer","data":[…]}` or an index-keyed object depending on the * backend's serializer. `EventSchema` accepts either β€” its payload fields are * unions that bottom out in `z.any()` β€” so nothing downstream would flag the - * mangled value; the runtime would hydrate garbage from it instead. + * mangled value; the runtime would hydrate garbage from it instead. A CBOR + * body round-trips the bytes intact and passes this check on its own merits, + * which is why a backend that attaches an event delta to a 412 encodes it that + * way. * * Refusing the delta is one-sided safe: the fallback full reload goes over a * frame-encoded path that returns real bytes. Deltas made only of * payload-less events (waits, hook disposal, attribute writes) keep the fast - * path. + * path whatever the encoding. */ function hasUnusablePayload(candidate: Record): boolean { const eventType = candidate.eventType; @@ -581,7 +646,7 @@ function hasUnusablePayload(candidate: Record): boolean { export function throwForErrorResponse( statusCode: number, responseHeaders: Record, - errorBody: string, + errorBody: string | Uint8Array, opName: string, url: string ): never { diff --git a/packages/world-vercel/src/events.test.ts b/packages/world-vercel/src/events.test.ts index 6ebdd6af97..ab1eb1632a 100644 --- a/packages/world-vercel/src/events.test.ts +++ b/packages/world-vercel/src/events.test.ts @@ -346,6 +346,48 @@ describe('createWorkflowRunEvent precondition snapshot wire fields', () => { agent.assertNoPendingInterceptors(); }); + it('renames eventCount to maxSlot', async () => { + // The runtime sends `eventCount` once a run's own ids are slot-shaped. It + // cannot ride under that name: the v4 meta already has an unrelated + // telemetry `eventCount`, so the backend would read a progress counter as + // a log position. + const agent = mockAgent(); + let capturedMeta: Record | undefined; + + agent + .get(ORIGIN) + .intercept({ + path: '/api/v4/runs/wrun_1/events/run_started', + method: 'POST', + }) + .reply( + 200, + (opts: { body?: unknown }) => { + capturedMeta = decodePostedMeta(opts.body); + return runStartedResponse(); + }, + { + headers: { + 'content-type': V4_FRAME_CONTENT_TYPE, + 'x-wf-event-id': 'evnt_1', + 'x-wf-run-id': 'wrun_1', + 'x-wf-created-at': '2026-06-10T00:00:00.000Z', + 'x-wf-max-events': '10000', + }, + } + ); + + await createWorkflowRunEvent( + 'wrun_1', + { eventType: 'run_started', specVersion: 2 } as AnyEventRequest, + { eventCount: 9 }, + { token: 'test-token', dispatcher: agent } + ); + + expect(capturedMeta?.maxSlot).toBe(9); + agent.assertNoPendingInterceptors(); + }); + it('never sends the snapshot on the legacy v1Compat path', async () => { // Pre-event-sourcing runs have no event log to fence, and the legacy // endpoint has no field for the snapshot: the params are dropped whole. diff --git a/packages/world-vercel/src/events.ts b/packages/world-vercel/src/events.ts index 12e98e49da..d5b981cef1 100644 --- a/packages/world-vercel/src/events.ts +++ b/packages/world-vercel/src/events.ts @@ -606,6 +606,11 @@ async function createWorkflowRunEventInner( stateUpdatedAt: params?.stateUpdatedAt, stateEventCount: params?.stateEventCount, ...(params?.stateCursor ? { stateCursor: params.stateCursor } : {}), + // Slot-identity snapshot. The runtime sends `eventCount` instead of the + // watermark triple once the run's own ids are slot-shaped; it rides as + // `maxSlot` because the v4 meta already has an unrelated telemetry + // `eventCount`. + ...(params?.eventCount !== undefined ? { maxSlot: params.eventCount } : {}), replayDivergenceCount: params?.replayDivergenceCount, occurredAt: params?.occurredAt ?? new Date(), // Opt-in inline-delta: forward the cursor the runtime held before diff --git a/packages/world-vercel/src/index.ts b/packages/world-vercel/src/index.ts index b9a38c3616..e94e98c07e 100644 --- a/packages/world-vercel/src/index.ts +++ b/packages/world-vercel/src/index.ts @@ -1,5 +1,5 @@ import type { World } from '@workflow/world'; -import { SPEC_VERSION_SUPPORTS_COMPRESSION } from '@workflow/world'; +import { SPEC_VERSION_SUPPORTS_SLOT_IDENTITY } from '@workflow/world'; import { createAnalytics } from './analytics.js'; import { createRunId, describeRun } from './create-run-id.js'; import { createGetEncryptionKeyForRun } from './encryption.js'; @@ -30,9 +30,12 @@ export function createWorld(config?: APIConfig): World { config?.projectConfig?.projectId || process.env.VERCEL_PROJECT_ID; return { - // Spec v5 adds client-side zstd/gzip payload compression. The server stores - // those payloads opaquely, and v5 remains a superset of v4 attributes. - specVersion: SPEC_VERSION_SUPPORTS_COMPRESSION, + // Spec v6 adds slot-numbered event ids on top of v5's client-side + // zstd/gzip payload compression. The version is what tells the backend + // which id scheme a run uses: it is stamped on `run_created` and read back + // on every later write, so a run created before v6 keeps its ULIDs even + // though this adapter now asks for slots. + specVersion: SPEC_VERSION_SUPPORTS_SLOT_IDENTITY, capabilities: { hookRetention: { active: true }, // workflow-server enforces the `stateUpdatedAt` optimistic-concurrency @@ -47,6 +50,11 @@ export function createWorld(config?: APIConfig): World { // Vercel deployments are atomic and immutable, so a deployment id names // one fixed build for its whole lifetime. deploymentAffinity: true, + // New runs get dense per-run slot event ids. Runs created before the + // backend adopted them keep their ULIDs; the scheme is pinned by the + // spec version stamped on each run, not by this flag, which only says + // what new runs get. + slotEventIds: true, // NOTE: the backend half of resumeHook()'s parallel fast path β€” that // the server enforces the `(runId, resumeId)` dedup constraint β€” is // NO LONGER a static world capability here. It is attested per-lookup by diff --git a/packages/world/src/events.ts b/packages/world/src/events.ts index e397d2c8b6..ba37c947ee 100644 --- a/packages/world/src/events.ts +++ b/packages/world/src/events.ts @@ -814,6 +814,38 @@ export interface CreateEventParams { * authoritative full reload, which is always correct. */ stateCursor?: string; + /** + * How many events the writer held in its loaded log when it decided to write + * this one β€” equivalently, the slot it expects to land on minus one. + * + * Only meaningful against a World that declares + * `WorldCapabilities.slotEventIds`, where slots are dense and 1-based so a + * count and a position are the same number. Such a World attempts + * `eventCount + 1`, and on contention **bumps** to the next free slot and + * commits there anyway β€” a stale count never rejects a write. What it does + * instead is report: when the committed slot is higher than the one asked + * for, the events occupying the skipped slots come back on the success + * response in {@link EventResult.events} / `cursor` / `hasMore`, so the + * writer learns exactly what it had not seen. + * + * This supersedes the {@link stateUpdatedAt} / {@link stateEventCount} / + * {@link stateCursor} triple for slot Worlds. That triple approximates a + * position with a ULID-time watermark plus a count of events at or below it, + * which is why a *complete but stale* prefix passes it: every event the + * writer holds is at or below its own watermark, so the count matches and no + * fence fires. A dense position has no such blind spot. Worlds without slots + * ignore this field and keep using the triple. + * + * A batch of writes issued from one snapshot starts from the same + * `eventCount`; they land on consecutive slots in whatever order the World + * serializes them, which is why they can stay a parallel fan-out instead of + * a chain of round-trips. The count a given write sends is the writer's + * position *at that moment*, so it advances mid-batch as reported events are + * folded back into the loaded log: a write issued after a sibling's + * bump-and-report already holds the slots that report named, and asks for a + * slot above them. + */ + eventCount?: number; /** * Timestamp for when the event occurred on the client side. Worlds that * support this can persist it separately from `createdAt`, which represents @@ -945,7 +977,7 @@ export type EventResult = { } & ( | { /** - * Events with data resolved. Three producers populate this: + * Events with data resolved. Four producers populate this: * * - On a `run_started` response: all events up to this point, so the * runtime can skip the initial `events.list` call and reduce TTFB. @@ -958,6 +990,13 @@ export type EventResult = { * log through the canonical `hook_received`, so the lazy hook queue * consumer can skip both the `run_started` write and the initial * `events.list`. + * - On any response from a slot-allocating World (see + * `WorldCapabilities.slotEventIds`) whose committed slot came out + * higher than the one {@link CreateEventParams.eventCount} asked for: + * the events occupying the slots that were skipped over, in slot + * order. This is the "report" half of bump-and-report β€” the write + * succeeded, and these are the events the writer had not seen when it + * decided to make it. */ events: Event[]; /** Pagination cursor for `events`, matching events.list semantics. */ diff --git a/packages/world/src/index.ts b/packages/world/src/index.ts index 68d0d2de65..50ce1b9a0a 100644 --- a/packages/world/src/index.ts +++ b/packages/world/src/index.ts @@ -111,16 +111,28 @@ export { PaginatedResponseSchema, StructuredErrorSchema, } from './shared.js'; +export { + EVENT_ID_BODY_LENGTH, + EVENT_ID_PREFIX, + eventIdToSlot, + FIRST_EVENT_SLOT, + isSlotBody, + isSlotEventId, + MAX_EVENT_SLOT, + slotToEventId, +} from './slot-identity.js'; export type { SpecVersion } from './spec-version.js'; export { isLegacySpecVersion, requiresNewerWorld, SPEC_VERSION_CURRENT, SPEC_VERSION_LEGACY, + SPEC_VERSION_MAX_SUPPORTED, SPEC_VERSION_SUPPORTS_ATTRIBUTES, SPEC_VERSION_SUPPORTS_CBOR_QUEUE_TRANSPORT, SPEC_VERSION_SUPPORTS_COMPRESSION, SPEC_VERSION_SUPPORTS_EVENT_SOURCING, + SPEC_VERSION_SUPPORTS_SLOT_IDENTITY, } from './spec-version.js'; export type * from './steps.js'; export { diff --git a/packages/world/src/interfaces.ts b/packages/world/src/interfaces.ts index e0184a0d2e..06a49b973e 100644 --- a/packages/world/src/interfaces.ts +++ b/packages/world/src/interfaces.ts @@ -415,6 +415,30 @@ export interface WorldCapabilities { * fail ordinary runs after a version bump. */ deploymentAffinity?: boolean; + + /** + * The World allocates **slot-numbered** event ids: `evnt_` plus the event's + * dense, 1-based position in its run's log, zero-padded to 26 characters + * (see `slot-identity.ts`). Two guarantees come with it, and the runtime + * relies on both: + * + * - **Density.** A run's slots are contiguous from 1, so the number of + * events a reader holds *is* the position of the last one. That is what + * makes {@link CreateEventParams.eventCount} a complete statement of the + * writer's snapshot, where the `stateUpdatedAt` / `stateEventCount` + * watermark pair could only approximate it. + * - **Bump and report.** A create never fails because its requested slot is + * taken. The World advances to the next free slot, commits there, and + * returns the events occupying the slots it skipped over on the success + * response (see {@link EventResult.events}). The writer learns its + * snapshot was stale without the write being rejected. + * + * A run's scheme is pinned by the run, not by this flag: it is readable off + * the shape of the run's own first event id, so a World that turns slots on + * keeps replaying its existing ULID-numbered runs unchanged. The capability + * only says what *new* runs get. + */ + slotEventIds?: boolean; } /** diff --git a/packages/world/src/slot-identity.test.ts b/packages/world/src/slot-identity.test.ts new file mode 100644 index 0000000000..63a7e85d30 --- /dev/null +++ b/packages/world/src/slot-identity.test.ts @@ -0,0 +1,93 @@ +import { ulid } from 'ulid'; +import { describe, expect, it } from 'vitest'; +import { + EVENT_ID_BODY_LENGTH, + EVENT_ID_PREFIX, + eventIdToSlot, + FIRST_EVENT_SLOT, + isSlotBody, + isSlotEventId, + slotToEventId, +} from './slot-identity.js'; +import { ulidToDate, validateUlidTimestamp } from './ulid.js'; + +describe('slotToEventId', () => { + it('mints a fixed-width id whose string order is slot order', () => { + const ids = [1, 2, 9, 10, 99, 100, 1000].map(slotToEventId); + for (const id of ids) { + expect(id).toHaveLength(EVENT_ID_PREFIX.length + EVENT_ID_BODY_LENGTH); + } + expect([...ids].sort()).toEqual(ids); + }); + + it('round-trips through eventIdToSlot', () => { + for (const slot of [FIRST_EVENT_SLOT, 7, 12345, Number.MAX_SAFE_INTEGER]) { + expect(eventIdToSlot(slotToEventId(slot))).toBe(slot); + } + }); + + it('refuses slots it cannot represent exactly', () => { + expect(() => slotToEventId(0)).toThrow(RangeError); + expect(() => slotToEventId(-1)).toThrow(RangeError); + expect(() => slotToEventId(1.5)).toThrow(RangeError); + expect(() => slotToEventId(Number.MAX_SAFE_INTEGER + 2)).toThrow( + RangeError + ); + }); +}); + +describe('isSlotEventId', () => { + it('reads an id however it is prefixed', () => { + const body = String(42).padStart(EVENT_ID_BODY_LENGTH, '0'); + expect(isSlotEventId(`evnt_${body}`)).toBe(true); + expect(isSlotEventId(`wevt_${body}`)).toBe(true); + expect(isSlotEventId(body)).toBe(true); + expect(eventIdToSlot(`wevt_${body}`)).toBe(42); + }); + + it('never mistakes a ULID for a slot', () => { + for (let i = 0; i < 100; i++) { + const id = ulid(); + expect(isSlotBody(id)).toBe(false); + expect(eventIdToSlot(`evnt_${id}`)).toBeNull(); + } + }); + + it('rejects bodies of the wrong shape', () => { + // Right length, but the timestamp region is not all zeros. + expect(isSlotBody('0000000001'.padEnd(EVENT_ID_BODY_LENGTH, '0'))).toBe( + false + ); + // Right prefix of zeros, but a non-digit in the counter region. + expect(isSlotBody('0'.repeat(EVENT_ID_BODY_LENGTH - 1) + 'A')).toBe(false); + // Wrong length. + expect( + isSlotBody('0'.repeat(EVENT_ID_BODY_LENGTH - 1) + '1'.repeat(2)) + ).toBe(false); + expect(isSlotBody('')).toBe(false); + }); +}); + +describe('time is never derived from a slot id', () => { + it('returns null rather than the epoch', () => { + // The trap this guards: `decodeTime` on a slot body succeeds and yields 0. + // A caller that took that at face value would date every event to 1970. + const body = slotToEventId(1).slice(EVENT_ID_PREFIX.length); + expect(ulidToDate(body)).toBeNull(); + expect( + ulidToDate(slotToEventId(999_999).slice(EVENT_ID_PREFIX.length)) + ).toBeNull(); + }); + + it('still decodes a real ULID', () => { + const id = ulid(); + expect(ulidToDate(id)?.getTime()).toBeGreaterThan(0); + }); + + it('fails validation instead of reporting 56 years of drift', () => { + const slotAsRunId = `wrun_${slotToEventId(1).slice(EVENT_ID_PREFIX.length)}`; + expect(validateUlidTimestamp(slotAsRunId, 'wrun_')).toMatch( + /is not a valid ULID/ + ); + }); +}); diff --git a/packages/world/src/slot-identity.ts b/packages/world/src/slot-identity.ts new file mode 100644 index 0000000000..a42283bba3 --- /dev/null +++ b/packages/world/src/slot-identity.ts @@ -0,0 +1,100 @@ +/** + * Slot-based event identity. + * + * An event id is `evnt_` followed by 26 characters. Historically that body was + * a ULID; a World that allocates *slots* instead writes the event's dense + * 1-based position in its run's log, as a zero-padded decimal. + * + * The padding is what makes this a drop-in change rather than a format break. + * Decimal digits are a subset of Crockford base32, and every id is still + * exactly 26 characters, so existing ULID validators accept a slot id, + * lexicographic ordering still matches creation order (fixed width, so string + * order is numeric order), and `eid:` cursors and range fences keep working + * untouched. + * + * The one thing that does *not* survive: a slot id's leading characters are + * zeros, so decoding it as a ULID timestamp yields the Unix epoch. Nothing may + * derive a time from an event id without first ruling out a slot id β€” see + * {@link isSlotBody} and the guard in `ulidToDate`. + */ + +/** Characters in an event id body, ULID or slot alike. */ +export const EVENT_ID_BODY_LENGTH = 26; + +/** + * Leading zeros a slot body must carry. + * + * This is the discriminator against a ULID: a ULID's first 10 characters + * encode milliseconds since the epoch, and `ulid()` never mints a zero + * timestamp. Requiring the same 10 characters to be `0` therefore separates + * the two schemes with no ambiguity, and caps a slot at 10^16 - 1 β€” far above + * any run's event count, and reduced further below to stay in safe-integer + * range. + */ +const SLOT_LEADING_ZEROS = 10; + +/** First slot in a run's log. Slots are 1-based and dense. */ +export const FIRST_EVENT_SLOT = 1; + +/** + * Largest representable slot. Bounded by JavaScript's safe-integer range + * rather than by the 16 significant digits the format allows, so a parsed slot + * is always exact. + */ +export const MAX_EVENT_SLOT = Number.MAX_SAFE_INTEGER; + +/** Canonical prefix for event ids. */ +export const EVENT_ID_PREFIX = 'evnt_'; + +/** + * Whether a 26-character event id *body* is a slot rather than a ULID. + * + * Takes the body, not the prefixed id, because the same test applies to event + * ids however they are spelled (`evnt_`, the legacy `wevt_`, or bare). + */ +export function isSlotBody(body: string): boolean { + if (body.length !== EVENT_ID_BODY_LENGTH) return false; + for (let i = 0; i < SLOT_LEADING_ZEROS; i++) { + if (body[i] !== '0') return false; + } + for (let i = SLOT_LEADING_ZEROS; i < EVENT_ID_BODY_LENGTH; i++) { + const code = body.charCodeAt(i); + if (code < 48 || code > 57) return false; + } + return true; +} + +/** Strips a `_` from an event id, if present. */ +function stripEventIdPrefix(eventId: string): string { + const underscore = eventId.indexOf('_'); + return underscore === -1 ? eventId : eventId.slice(underscore + 1); +} + +/** Whether a (possibly prefixed) event id is slot-numbered. */ +export function isSlotEventId(eventId: string): boolean { + return isSlotBody(stripEventIdPrefix(eventId)); +} + +/** + * Formats a slot as a prefixed event id. + * + * @throws if the slot is outside the representable range β€” a caller that + * overflows must fail loudly rather than mint an id that sorts wrong. + */ +export function slotToEventId(slot: number): string { + if (!Number.isSafeInteger(slot) || slot < FIRST_EVENT_SLOT) { + throw new RangeError(`Invalid event slot: ${slot}`); + } + return `${EVENT_ID_PREFIX}${String(slot).padStart(EVENT_ID_BODY_LENGTH, '0')}`; +} + +/** + * Reads the slot out of a (possibly prefixed) event id, or null when the id is + * not slot-numbered. + */ +export function eventIdToSlot(eventId: string): number | null { + const body = stripEventIdPrefix(eventId); + if (!isSlotBody(body)) return null; + const slot = Number(body); + return Number.isSafeInteger(slot) && slot >= FIRST_EVENT_SLOT ? slot : null; +} diff --git a/packages/world/src/spec-version.test.ts b/packages/world/src/spec-version.test.ts index 880c43180d..da9d9a3f79 100644 --- a/packages/world/src/spec-version.test.ts +++ b/packages/world/src/spec-version.test.ts @@ -4,8 +4,10 @@ import { requiresNewerWorld, SPEC_VERSION_CURRENT, SPEC_VERSION_LEGACY, + SPEC_VERSION_MAX_SUPPORTED, SPEC_VERSION_SUPPORTS_ATTRIBUTES, SPEC_VERSION_SUPPORTS_COMPRESSION, + SPEC_VERSION_SUPPORTS_SLOT_IDENTITY, } from './spec-version.js'; describe('spec version constants', () => { @@ -13,6 +15,19 @@ describe('spec version constants', () => { expect(SPEC_VERSION_CURRENT).toBe(SPEC_VERSION_SUPPORTS_COMPRESSION); expect(SPEC_VERSION_SUPPORTS_COMPRESSION).toBe(5); }); + + it('the readable ceiling is the slot-identity version', () => { + // The default a World stamps and the highest version this SDK can read + // are separate dials. Slot identity is above the default on purpose: only + // a World that actually allocates slots opts into it. + expect(SPEC_VERSION_SUPPORTS_SLOT_IDENTITY).toBe(6); + expect(SPEC_VERSION_MAX_SUPPORTED).toBe( + SPEC_VERSION_SUPPORTS_SLOT_IDENTITY + ); + expect(SPEC_VERSION_MAX_SUPPORTED).toBeGreaterThanOrEqual( + SPEC_VERSION_CURRENT + ); + }); }); describe('requiresNewerWorld', () => { @@ -24,13 +39,20 @@ describe('requiresNewerWorld', () => { expect(requiresNewerWorld(null)).toBe(false); }); - it('rejects runs newer than the current spec version', () => { + it('accepts a slot-identity run even though it is above the default', () => { + // world-vercel stamps this version on the runs it creates. Testing + // against SPEC_VERSION_CURRENT instead of the ceiling would make this SDK + // reject the runs its own adapter just wrote. + expect(requiresNewerWorld(SPEC_VERSION_SUPPORTS_SLOT_IDENTITY)).toBe(false); + }); + + it('rejects runs newer than the highest supported spec version', () => { // This is the contract that protects older SDKs from compressed // payloads they cannot decode: a spec-5 run read by an SDK whose - // SPEC_VERSION_CURRENT is 4 fails this check up front (with - // RunNotSupportedError at the storage layer) instead of failing on - // individual compressed payloads. - expect(requiresNewerWorld(SPEC_VERSION_CURRENT + 1)).toBe(true); + // ceiling is 4 fails this check up front (with RunNotSupportedError at + // the storage layer) instead of failing on individual compressed + // payloads. + expect(requiresNewerWorld(SPEC_VERSION_MAX_SUPPORTED + 1)).toBe(true); }); it('simulates a v4 reader rejecting a compression-era run', () => { diff --git a/packages/world/src/spec-version.ts b/packages/world/src/spec-version.ts index cf516b772b..21112c7553 100644 --- a/packages/world/src/spec-version.ts +++ b/packages/world/src/spec-version.ts @@ -31,13 +31,51 @@ export const SPEC_VERSION_SUPPORTS_ATTRIBUTES = 4 as SpecVersion; */ export const SPEC_VERSION_SUPPORTS_COMPRESSION = 5 as SpecVersion; +/** + * Runs at this spec version get slot-numbered event ids: `evnt_` followed by a + * zero-padded decimal position, dense and contiguous from 1 within one run. + * + * This exists for Worlds that cannot read a run's scheme off its own storage. + * `world-local` and `world-postgres` own the counter that mints the ids, so + * they know per run which scheme it started under. `world-vercel` writes + * through an API whose allocator has to make that decision on each request, + * and the spec version stamped on `run_created` is what carries it. A run + * created before the backend adopted slots stays on ULIDs for its whole life + * because its stamped version is below this one. + */ +export const SPEC_VERSION_SUPPORTS_SLOT_IDENTITY = 6 as SpecVersion; + /** * Current spec version (event-sourced architecture with native attributes * and compressed payloads). + * + * Deliberately NOT bumped for slot-numbered event ids. Slot numbering is a + * property of a run's whole log rather than of an individual event, and it is + * already self-describing: a run's scheme is readable from the shape of its + * own first event id (see `isSlotEventId`), so a World that owns its own id + * allocation needs no version negotiation to pin one. Bumping this constant + * would stamp the new version on every World + * including ones that have not adopted slots yet, which is exactly the + * cross-version breakage the pin exists to avoid. A World that does allocate + * slots declares the higher version itself (see `world-vercel`), and + * `SPEC_VERSION_MAX_SUPPORTED` is what keeps this reader from rejecting the + * runs it produces. */ export const SPEC_VERSION_CURRENT = SPEC_VERSION_SUPPORTS_COMPRESSION as SpecVersion; +/** + * The highest spec version this SDK can read. + * + * Distinct from `SPEC_VERSION_CURRENT`, which is the *default* a World stamps + * on runs it creates. A World may declare a higher version than the default, + * so the "was this run made by a newer SDK?" test has to be against the + * ceiling: comparing against the default would make the SDK reject runs its + * own adapters just created. + */ +export const SPEC_VERSION_MAX_SUPPORTED = + SPEC_VERSION_SUPPORTS_SLOT_IDENTITY as SpecVersion; + /** * Check if a spec version is legacy (<= SPEC_VERSION_LEGACY or undefined). * Legacy runs require different handling - they use direct entity mutation @@ -55,7 +93,7 @@ export function isLegacySpecVersion(v: number | undefined | null): boolean { } /** - * Check if a spec version requires a newer world (> SPEC_VERSION_CURRENT). + * Check if a spec version requires a newer world (> SPEC_VERSION_MAX_SUPPORTED). * This happens when a run was created by a newer SDK version. * * @param v - The spec version number, or undefined/null for legacy runs @@ -63,5 +101,5 @@ export function isLegacySpecVersion(v: number | undefined | null): boolean { */ export function requiresNewerWorld(v: number | undefined | null): boolean { if (v === undefined || v === null) return false; - return v > SPEC_VERSION_CURRENT; + return v > SPEC_VERSION_MAX_SUPPORTED; } diff --git a/packages/world/src/ulid.ts b/packages/world/src/ulid.ts index 1ee1b7b47c..3ba1df2c99 100644 --- a/packages/world/src/ulid.ts +++ b/packages/world/src/ulid.ts @@ -1,5 +1,6 @@ import { decodeTime } from 'ulid'; import { z } from 'zod'; +import { isSlotBody } from './slot-identity.js'; const UlidSchema = z.string().ulid(); @@ -36,8 +37,18 @@ export const DEFAULT_TIMESTAMP_THRESHOLD_MS = /** * Extracts a Date from a ULID string, or null if the string is not a valid ULID. + * + * A slot-numbered event id is syntactically a valid ULID (26 zero-padded + * decimal digits are all Crockford characters) whose timestamp component is + * zero, so decoding one would silently yield the Unix epoch. Slots carry no + * time at all, so this returns null for them and callers fall back to a real + * `createdAt`. See `slot-identity.ts`. */ export function ulidToDate(maybeUlid: string): Date | null { + if (isSlotBody(maybeUlid)) { + return null; + } + const ulid = UlidSchema.safeParse(maybeUlid); if (!ulid.success) { return null; diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index 18058896ee..392d99262c 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -1476,6 +1476,37 @@ importers: specifier: 'catalog:' version: 4.1.10(@opentelemetry/api@1.9.1)(@types/node@22.19.0)(@vitest/coverage-v8@4.1.10)(jsdom@26.1.0)(vite@8.1.3(@types/node@22.19.0)(esbuild@0.25.12)(jiti@2.7.0)(terser@5.44.0)(tsx@4.20.6)(yaml@2.9.0)) + packages/world-sim: + dependencies: + '@workflow/builders': + specifier: workspace:* + version: link:../builders + '@workflow/core': + specifier: workspace:* + version: link:../core + '@workflow/errors': + specifier: workspace:* + version: link:../errors + '@workflow/utils': + specifier: workspace:* + version: link:../utils + '@workflow/world': + specifier: workspace:* + version: link:../world + devDependencies: + '@types/node': + specifier: 'catalog:' + version: 22.19.0 + '@workflow/tsconfig': + specifier: workspace:* + version: link:../tsconfig + typescript: + specifier: 'catalog:' + version: 6.0.3 + vitest: + specifier: 'catalog:' + version: 4.1.10(@opentelemetry/api@1.9.1)(@types/node@22.19.0)(@vitest/coverage-v8@4.1.10)(jsdom@26.1.0)(vite@8.1.3(@types/node@22.19.0)(esbuild@0.28.1)(jiti@2.7.0)(terser@5.44.0)(tsx@4.20.6)(yaml@2.9.0)) + packages/world-testing: dependencies: '@hono/node-server': @@ -2282,6 +2313,24 @@ importers: specifier: 'catalog:' version: 4.3.6 + workbench/sim-world: + devDependencies: + '@types/node': + specifier: 'catalog:' + version: 22.19.0 + '@workflow/tsconfig': + specifier: workspace:* + version: link:../../packages/tsconfig + '@workflow/world-sim': + specifier: workspace:* + version: link:../../packages/world-sim + typescript: + specifier: 'catalog:' + version: 6.0.3 + workflow: + specifier: workspace:* + version: link:../../packages/workflow + workbench/sveltekit: dependencies: '@ai-sdk/react': diff --git a/workbench/fastify/.gitignore b/workbench/fastify/.gitignore index 107101d13a..4f99d9f581 100644 --- a/workbench/fastify/.gitignore +++ b/workbench/fastify/.gitignore @@ -25,3 +25,6 @@ vite.config.ts.timestamp-* # Workflows _workflows.ts /.swc + +# Copied from index.html by `copy:index` so nitro can serve it statically +/public/index.html diff --git a/workbench/fastify/package.json b/workbench/fastify/package.json index 48758cad2a..f52cd7dfc9 100644 --- a/workbench/fastify/package.json +++ b/workbench/fastify/package.json @@ -7,7 +7,7 @@ "main": "index.js", "scripts": { "generate:workflows": "node ../scripts/generate-workflows-registry.js", - "predev": "pnpm generate:workflows", + "predev": "pnpm generate:workflows && pnpm copy:index", "copy:index": "mkdir -p public && cp index.html public/index.html", "prebuild": "pnpm generate:workflows && pnpm copy:index", "postbuild": "rm -f public/index.html", diff --git a/workbench/nextjs-turbopack/vercel.json b/workbench/nextjs-turbopack/vercel.json index 22da2a3327..ff2e7944b0 100644 --- a/workbench/nextjs-turbopack/vercel.json +++ b/workbench/nextjs-turbopack/vercel.json @@ -5,8 +5,7 @@ } }, "env": { - "WORKFLOW_PUBLIC_MANIFEST": "1", - "WORKFLOW_PER_KIND_CORRELATION_IDS": "1" + "WORKFLOW_PUBLIC_MANIFEST": "1" }, "regions": [ "iad1", diff --git a/workbench/sim-world/.gitignore b/workbench/sim-world/.gitignore new file mode 100644 index 0000000000..86f7b2dd6c --- /dev/null +++ b/workbench/sim-world/.gitignore @@ -0,0 +1,6 @@ +.workflow-sim/ +/.swc +# Written by `pnpm test` (--summary-file) and by CI runs that pass +# --detail-file; both are reports about a run, not inputs to one. +/sim-summary.md +/sim-detail.txt diff --git a/workbench/sim-world/README.md b/workbench/sim-world/README.md new file mode 100644 index 0000000000..ea48165e1d --- /dev/null +++ b/workbench/sim-world/README.md @@ -0,0 +1,261 @@ +# sim-world workbench + +Worked examples for `@workflow/world-sim`: workflows written to make ordering +visible, and a book of scenarios that pin down exactly when external input +arrives. + +```bash +pnpm sim # play every scenario, print every event stream +pnpm sim hook # only scenarios whose id or name contains "hook" +pnpm sim in-flight-after-decision # one scenario, by id +``` + +Exits non-zero if any scenario misses an expectation or trips a consistency +check, so it doubles as the package's integration test. + +This README is about **adding a scenario**. The API a script is written in β€” +writers, advances, withholdings β€” is the +[API reference](../../packages/world-sim/README.md#api-reference); how the +simulator works and how to change it is the rest of +[`packages/world-sim/README.md`](../../packages/world-sim/README.md), and the +internals are [`DESIGN.md`](../../packages/world-sim/DESIGN.md). + +## Adding a scenario + +One scenario, one file in [`scenarios/`](./scenarios), named after its id. +Copy the file next door and change what differs β€” that is the whole workflow, +and the book is split this way so that it is. + +```ts +// scenarios/hook-at-step-started.ts +import type { ScenarioSpec } from '@workflow/world-sim'; + +export const scenario: ScenarioSpec = { + id: 'hook-at-step-started', + name: 'hook arrives inside the step_started commit', + description: 'The hook payload is written after step_started is durable …', + workflow: 'approvalWorkflow', + input: ['doc-1'], + script: async (sim) => { + const wf = sim.writer.orchestrator(); + await wf.runToEventCommitted('step_started', 'reserveInventory'); + await sim.deliverHook('approval:doc-1', { approved: true, reviewer: 'ada' }); + await wf.release(); + }, + expect: { + status: 'completed', + output: { status: 'settled:reserved:doc-1', reviewer: 'ada' }, + }, +}; +``` + +Then import it in [`scenarios/index.ts`](./scenarios/index.ts) and place it in +the `scenarios` array. Order is the only thing that file decides: simplest +first, and each pair of near-identical scenarios adjacent, so a reader meets a +distinction right after the thing it is a distinction from. Put yours next to +the one it is a variation of. + +The id is stable and hyphenated; it is what a commit message or a bug report +cites and what the command-line filter matches first. The `name` beside it is +prose and free to be reworded. + +The workflow named by `workflow` must be exported from +[`workflows/index.ts`](./workflows/index.ts) β€” all of them live in that one +file because a scenario is read together with the branch it steers. Prefer +reusing one; a new workflow is only worth it when the shape you need to steer +does not exist yet. + +### The shape of a script + +Every script is the same three steps: **hold a writer at a named point, act +while it is held, let it go.** + +```ts +const wf = sim.writer.orchestrator(); +await wf.runToEventCommitted('step_started', 'reserveInventory'); +await sim.deliverHook('approval:doc-1', { approved: true }); +await wf.release(); +``` + +Because the writer is held *inside* the world call, everything the script does +in between lands in the log before that writer is resumed. That is the entire +point of the writer API: the interleaving is stated, not raced for. + +Every advance and everything a script can do while one is held is in the +[API reference](../../packages/world-sim/README.md#api-reference). Four things +from it come up on the first scenario you write: + +- **Name the right writer.** `step_started`, `wait_created`, `hook_created` and + the run's own decisions belong to `sim.writer.orchestrator()`. A step's + `step_completed` / `step_failed` belongs to that step body β€” + `sim.writer.step('reserveInventory')`, or `sim.writer.anyStep()` for whichever + gets there first. Naming the wrong one is a wait that times out, so the + failure is loud, but knowing the rule saves the trip. +- **Pick the right advance.** `runToEventCommitted` is what most scenarios want. + Reach for `runToEventProduced` when the point is that a write committed during + the hold sorts *ahead* of the held event, and for `sim.beginHookDelivery` when + it has to sort *behind* one. +- **Calling an advance starts watching; awaiting it waits for the hold.** To + hold two writers at once, call both, then await both. +- **`runTo` is level-triggered.** Asking for a point that has already gone by + is an error, not a wait that never ends. +- **Start B's watch before releasing A.** A released writer can reach the next + point within the same turn, and a watch started afterwards has missed it. + +And one thing the advances cannot do at all: a held writer stops the scheduler, +so virtual time stops with it and no timer can fire while anything is held. If +the interleaving you need is *a timer firing while a step result is +outstanding*, no arrangement of holds will reach it. `sim.deliverQueued` is the +way out β€” it delivers a queued message from inside the script, concurrently with +the hold. See +[the API reference](../../packages/world-sim/README.md#deliverqueued-and-why-it-is-not-an-advance) +for the shape, and +[`unclaimed-payload-under-fork.ts`](./scenarios/unclaimed-payload-under-fork.ts) +for it in use. + +### What to assert, and what not to + +Two different instruments, for two different things: + +- **`sim.check`** asserts a *sentence about the middle of the run* β€” "the live + pass decided the fork without the hook". It is the only way to pin down a + fact that exists at one instant and is gone by the end. +- **`expect`** asserts the run's outcome: `status`, and `output` when the + output is the point. + +And one rule that matters more than either: **do not restate an expectation per +world.** A scenario is one sequence of advances; the only thing a flag like +`--append-only` changes is what a read returns. An expectation that has to be +written twice is pinning a *consequence of the reads* rather than a property of +the run, and a scenario that branches its tempo on `sim.appendOnlyLog` is two +scenarios wearing one id. + +What catches the fault in every world is the invariant the runner checks for +free: **a run's log must replay back into that run.** So when a flag decides +which branch a run takes, report the branch with `sim.note` and assert only +what holds either way β€” usually `status`, plus the replay check you get without +asking. Reading `sim.appendOnlyLog` to *phrase* a check's sentence correctly is +fine and encouraged; reading it to choose a different tempo is not. + +There is deliberately no way to expect a violation. A scenario states the +outcome the run should have reached and stays red until the runtime gets there. + +### Per-scenario world flags + +`preconditionGuard`, `countGuard` and `appendOnlyLog` on the spec pick the world +this scenario plays in. The usual reason to set one is a **paired scenario**: +the red one and the same tempo with a fix armed, one flag apart, so the diff is +the argument. The command-line flags below override the spec for a whole run. + +## Flags + +| flag | effect | +| --- | --- | +| `--verbose` | include queue deliveries in the trace | +| `--color` / `--no-color` | force colour on through a pipe / off. Default: on for a terminal, off otherwise, so `pnpm sim > out.txt` is already diffable | +| `--append-only` / `--no-append-only` | play against an append-only log, or force production behaviour back on | +| `--fence` / `--no-fence` | force the optimistic-concurrency fence on or off for every scenario | +| `--report-only` | print every failure, exit 0 anyway | +| `--summary-file ` | one collapsed `
` β€” the count on the visible line, the table behind it β€” for a PR comment or `$GITHUB_STEP_SUMMARY` | +| `--detail-file ` | the full trace, colour forced off, as a CI artifact | +| `--title ` | heading for the summary file, so two of them in one comment are told apart by more than their chips line | + +Two of these are measurements rather than conveniences. + +**`--append-only`** moves every event's position from its handler's mint to its +commit, which is the one change that makes a stale read impossible: the log can +be behind, never wrong. Running with and without it is how you tell which of +the reds that change would actually close. Today: **35 pass / 6 violations** +mint-ordered, **41 pass / 0 violations** append-only. + +The one red it does *not* close is `unclaimed-payload-under-fork`, and that is +the point of it: no log position is wrong there, the runtime hands two +resolutions to the workflow in the order the log did not record. It is the only +scenario in the book that is red in both worlds. + +**`--no-fence`** turns the fence off everywhere, asking whether anything relies +on it. It is a diagnostic, not a world β€” **read the violation count, not the +pass count**, because a scenario whose whole point is that the guard fired +asserts exactly that and fails by design when you disarm it +(`in-flight-before-decision-counted` is the one that does this today). +Measured: **6 β†’ 8** violations mint-ordered, so it is load-bearing there; +**0 β†’ 0** append-only, so it is dead weight once positions are assigned at +commit. + +## In CI + +[`.github/workflows/world-sim.yml`](../../.github/workflows/world-sim.yml) +plays the book on every pull request, once per world, and posts both summaries +as one sticky comment β€” four lines until you open something: + +``` +## Sim World + +Simulated world deterministic testing for races. [Traces](…) + +β–Έ 🟠 Mint-ordered log β€” 6 fail of 41 total +β–Έ 🟒 Append-only log β€” 0 fail of 41 total +``` + +**It never blocks a merge**: those scenarios are red on purpose, so a lane that +gated on them would be red on every PR and read as broken rather than as +informative. What it publishes is the pair of counts, and the thing to look at +is whether they still say 6 and 0. + +That is also why `pnpm test` in this package is `--report-only` while `pnpm sim` +stays strict β€” a recursive `pnpm -r test` should not go red for the known reds, +but someone running the book deliberately wants the exit code. + +## Reading the output + +Events in the printed stream are referred to by **log position** β€” `#12` is the +twelfth event in the durable log, `@7` the resource created at position 7 β€” and +the trace prints in *commit* order, so the numbers count backwards exactly where +the log and the execution disagree. See +[`packages/world-sim/README.md`](../../packages/world-sim/README.md#reading-the-output). + +## What the scenarios show + +The first three run the **same workflow with the same input** and differ only in +when the approval hook is delivered β€” inside the `step_started` commit, inside +the `step_completed` commit, or inside the `hook_created` commit. Same result, +three different event logs. Diff them against each other; that difference is +what a real deployment leaves to chance. + +Two of them ("writers: …") make the underlying claim explicit: the two step +bodies of a single delivery are separately steerable writers to one log, and +holding one does not freeze the other. + +The rest cover the properties that make scenarios usable as tests: a hook +racing a deadline (both branches, on demand), a thirty-day sleep that costs +microseconds, a step that retries twice, cancellation landing mid-step, and a +hook that never arrives β€” which is reported as a stall naming the undelivered +token rather than hanging the run. + +## Red scenarios + +Some scenarios fail, on purpose and by construction, and `pnpm sim` exits +non-zero because of them. They are reproductions of corruptions the runtime can +still produce: each states the outcome the run should have reached β€” the branch +its own durable log implies β€” and fails until the runtime gets there. The +failure line names both sides, e.g. `expected "afterSlow:doc-26", got +"afterFast:doc-26"`. + +So a red is an open bug, not a recorded observation, and it goes green when the +bug is fixed rather than when the bug is seen once more. Which means the count +is the thing to watch, in either direction: one more is a regression, one fewer +means a scenario is ready to retire. + +Run the book to see the current set β€” this file deliberately does not keep a +list, because a list here is a second copy of something the book already says +exactly, and it is the copy that goes stale. The analysis that is *not* +re-derivable from a run β€” which guard closes which shape, which of those guards +is armed in production and which is dark β€” is in +[`DESIGN.md`](../../packages/world-sim/DESIGN.md#the-six). + +## Requirements + +`run.ts` and the scenario book are TypeScript executed directly by Node's type +stripping, which needs Node >= 22.18 (the version pinned in `.node-version`). +Every workflow under test is compiled by the normal SDK build pipeline, exactly +as a deployment would compile it. diff --git a/workbench/sim-world/package.json b/workbench/sim-world/package.json new file mode 100644 index 0000000000..18c8ca63a1 --- /dev/null +++ b/workbench/sim-world/package.json @@ -0,0 +1,22 @@ +{ + "name": "@workflow/sim-world-workbench", + "private": true, + "type": "module", + "version": "0.0.1", + "license": "Apache-2.0", + "scripts": { + "sim": "node run.ts", + "test": "node run.ts --report-only --summary-file sim-summary.md", + "clean": "rm -rf .workflow-sim sim-summary.md" + }, + "devDependencies": { + "@types/node": "catalog:", + "@workflow/tsconfig": "workspace:*", + "@workflow/world-sim": "workspace:*", + "typescript": "catalog:", + "workflow": "workspace:*" + }, + "engines": { + "node": ">=22.18" + } +} diff --git a/workbench/sim-world/run.ts b/workbench/sim-world/run.ts new file mode 100644 index 0000000000..5adb85f90e --- /dev/null +++ b/workbench/sim-world/run.ts @@ -0,0 +1,205 @@ +/** + * Play the scenario book and print each event stream. + * + * pnpm sim # every scenario + * pnpm sim hook # scenarios whose id or name contains "hook" + * pnpm sim in-flight-after-decision # one scenario, by id + * pnpm sim --verbose # include queue deliveries in the trace + * pnpm sim --no-color # plain ASCII, e.g. for a golden file + * pnpm sim --append-only # play against an append-only log + * pnpm sim --no-fence # play with the optimistic-concurrency fence off + * pnpm sim --report-only # print failures but exit 0 + * pnpm sim --summary-file s.md # markdown counts + table, for a PR comment + * pnpm sim --detail-file d.txt # the full trace, colour-free, as an artifact + * pnpm sim --title 'Append-only' # heading for the summary file + * + * `--append-only` moves every event's position from its handler's mint to its + * commit, which is the one change that makes a stale read impossible: the log + * can be behind, never wrong. Six scenarios in the book fail today because it + * is *not* how production works, so running with and without it β€” and diffing + * β€” is how you tell which of those failures the change would actually close. + * `--no-append-only` forces the production behaviour back on for a scenario + * that asked for the flag itself. + * + * `--no-fence` turns off the optimistic-concurrency fence (both halves β€” the + * count guard is evaluated inside the same predicate) for every scenario that + * asked for it. The fence rejects a write whose snapshot predates an + * out-of-band event: a write that an extended prefix invalidated. So a book + * that scores the same with the fence off is a book in which no emitter is + * prefix-sensitive, and the fence is protecting against nothing. Anything that + * goes red only under `--no-fence` names the exception. `--fence` forces it on. + * + * `--no-fence` is a diagnostic, not a world: read the *violation* count, not + * the pass count. A scenario whose whole point is that the guard fired asserts + * exactly that with `sim.check`, so turning the guard off fails it by design β€” + * `in-flight-before-decision-counted` is the one that does this today. The + * violation count is the number that means something. + * + * Colour is on by default when stdout is a terminal and off otherwise, so + * `pnpm sim > out.txt` already produces a diffable file; `--no-color` and + * `NO_COLOR` force it off, `--color` forces it on through a pipe. + * + * Exits non-zero when any scenario fails an expectation or trips a + * consistency check, so this doubles as a test command. `--report-only` keeps + * every one of those messages and exits 0 anyway, which is what a CI job that + * wants to *publish* the book's current state rather than gate on it needs. + */ + +import { writeFile } from 'node:fs/promises'; +import { resolve } from 'node:path'; +import { fileURLToPath } from 'node:url'; +import { + loadFlowHandler, + renderMarkdownSummary, + renderScenario, + renderSummary, + runScenario, + type ScenarioResult, +} from '@workflow/world-sim'; +import { buildSimBundle } from '@workflow/world-sim/build'; +import { scenarios } from './scenarios/index.ts'; + +const args = process.argv.slice(2); +const verbose = args.includes('--verbose'); +const color = args.includes('--no-color') + ? false + : args.includes('--color') + ? true + : undefined; +// `undefined` leaves it to each spec, which is not the same as `false` β€” +// see `RunScenarioOptions.appendOnlyLog`. +const appendOnlyLog = args.includes('--no-append-only') + ? false + : args.includes('--append-only') + ? true + : undefined; +// Same tri-state as above, and for the same reason: a scenario that turns the +// fence on itself is the normal case, so `undefined` has to mean "leave it to +// the spec" rather than "off". +const preconditionGuard = args.includes('--no-fence') + ? false + : args.includes('--fence') + ? true + : undefined; +const reportOnly = args.includes('--report-only'); +const summaryFile = pathValue('--summary-file'); +const detailFile = pathValue('--detail-file'); +// Names the summary's heading. A CI job plays the book once per world and +// concatenates the two files into one comment, where two headings reading +// `world-sim scenario book` would leave the chips line as the only way to tell +// which half you are looking at. +const summaryTitle = flagValue('--title') ?? 'world-sim scenario book'; + +/** + * Read `--flag value`. Anything a flag consumes is removed from `args` before + * the filters are taken, so `--summary-file s.md` does not also select every + * scenario whose id contains "s.md" (which is none, but the next path won't be + * so lucky). + */ +function flagValue(name: string): string | undefined { + const at = args.indexOf(name); + if (at === -1) return undefined; + const value = args[at + 1]; + if (value === undefined || value.startsWith('--')) { + console.error(`${name} needs a value`); + process.exit(1); + } + args.splice(at, 2); + return value; +} + +/** `flagValue`, resolved against the cwd β€” for the flags that name a file. */ +function pathValue(name: string): string | undefined { + const value = flagValue(name); + return value === undefined ? undefined : resolve(value); +} + +const filters = args.filter((a) => !a.startsWith('--')); + +const cwd = fileURLToPath(new URL('.', import.meta.url)); + +// Matched against the id first, because that is the handle a bug report or a +// commit message will have used; the prose name stays searchable as a +// fallback so `pnpm sim deadline` keeps working. +const selected = scenarios.filter( + (s) => + filters.length === 0 || + filters.some((f) => s.id.includes(f) || s.name.includes(f)) +); + +if (selected.length === 0) { + console.error(`No scenario matches ${filters.join(', ')}`); + process.exit(1); +} + +console.log('Building workflow bundle...'); +const bundle = await buildSimBundle({ cwd, dirs: ['workflows'] }); +const handler = await loadFlowHandler(bundle.flowBundlePath); +console.log( + `Loaded ${Object.keys(bundle.manifest.workflows ?? {}).length} workflow module(s)\n` +); + +const results: ScenarioResult[] = []; +// The detail artifact is built from the same renders as the console, only with +// colour forced off, so the file and the terminal can never disagree about +// what happened β€” they are the same call with one option flipped. +const detail: string[] = []; + +// Scenarios run strictly one at a time: each installs a virtual clock and a +// process-global World, both of which are singletons. +for (const spec of selected) { + const result = await runScenario(spec, { + handler, + workflowIds: bundle.workflowIds, + appendOnlyLog, + preconditionGuard, + }); + results.push(result); + console.log(renderScenario(result, { verbose, color })); + console.log(''); + if (detailFile) { + // Always verbose in the artifact: it is read when the summary was not + // enough, and a truncated artifact just sends the reader back to a laptop. + detail.push(renderScenario(result, { verbose: true, color: false }), ''); + } +} + +console.log(renderSummary(results, { color })); + +if (detailFile) { + detail.push(renderSummary(results, { color: false }), ''); + await writeFile(detailFile, detail.join('\n'), 'utf8'); + console.log(`Wrote detail to ${detailFile}`); +} + +if (summaryFile) { + await writeFile( + summaryFile, + renderMarkdownSummary(results, { + title: summaryTitle, + // Say which world, always β€” including when it is the default one. A + // summary file outlives the command line that produced it, and two of + // these sitting side by side in a PR comment are only comparable if each + // one states its own conditions. + chips: [ + `log=${appendOnlyLog === true ? 'append-only' : 'mint-ordered'}`, + `fence=${ + preconditionGuard === undefined + ? 'per-spec' + : preconditionGuard + ? 'forced-on' + : 'off' + }`, + ], + detailPath: detailFile, + }), + 'utf8' + ); + console.log(`Wrote summary to ${summaryFile}`); +} + +const failed = !results.every((r) => r.ok); +if (failed && reportOnly) { + console.log('\n--report-only: exiting 0 despite the failures above.'); +} +process.exit(failed && !reportOnly ? 1 : 0); diff --git a/workbench/sim-world/scenarios/attr-from-step-body.ts b/workbench/sim-world/scenarios/attr-from-step-body.ts new file mode 100644 index 0000000000..cce40c16b2 --- /dev/null +++ b/workbench/sim-world/scenarios/attr-from-step-body.ts @@ -0,0 +1,23 @@ +import type { ScenarioSpec } from '@workflow/world-sim'; + +export const scenario: ScenarioSpec = { + id: 'attr-from-step-body', + name: 'attr: a step writes run state while a hook lands mid-flight', + description: + 'The attr_set comes from step context (writer type "step", committed ' + + 'inline, no correlationId dedupe), so unlike the orchestrator path its ' + + 'log position is decided by step timing rather than by suspension. The ' + + 'writer column in the trace names the step body that wrote it.', + workflow: 'stepAttributeWorkflow', + input: ['doc-16'], + script: async (sim) => { + const recorder = sim.writer.step('probeAndRecord'); + await recorder.runToEventCommitted('attr_set'); + await sim.deliverHook('stepattr:doc-16', { approved: true }); + await recorder.release(); + }, + expect: { + status: 'completed', + output: 'recorded:doc-16|probed:doc-16|yes', + }, +}; diff --git a/workbench/sim-world/scenarios/attr-hook-after-step.ts b/workbench/sim-world/scenarios/attr-hook-after-step.ts new file mode 100644 index 0000000000..ae84301989 --- /dev/null +++ b/workbench/sim-world/scenarios/attr-hook-after-step.ts @@ -0,0 +1,18 @@ +import type { ScenarioSpec } from '@workflow/world-sim'; + +export const scenario: ScenarioSpec = { + id: 'attr-hook-after-step', + name: 'attr: hook lands AFTER the concurrent step completes', + workflow: 'concurrentAttributeWorkflow', + input: ['doc-15'], + script: async (sim) => { + const probe = sim.writer.step('probe'); + await probe.runToEventCommitted('step_completed'); + await sim.deliverHook('attr:doc-15', { approved: false }); + await probe.release(); + }, + expect: { + status: 'completed', + output: 'probed:doc-15/rejected', + }, +}; diff --git a/workbench/sim-world/scenarios/attr-hook-before-step.ts b/workbench/sim-world/scenarios/attr-hook-before-step.ts new file mode 100644 index 0000000000..0565bccd66 --- /dev/null +++ b/workbench/sim-world/scenarios/attr-hook-before-step.ts @@ -0,0 +1,21 @@ +import type { ScenarioSpec } from '@workflow/world-sim'; + +export const scenario: ScenarioSpec = { + id: 'attr-hook-before-step', + name: 'attr: hook lands BEFORE the concurrent step completes', + description: + 'The hook-gated branch writes its attr_set ahead of the other branch ' + + "step's result in the log.", + workflow: 'concurrentAttributeWorkflow', + input: ['doc-14'], + script: async (sim) => { + const probe = sim.writer.step('probe'); + await probe.runToEventProduced('step_completed'); + await sim.deliverHook('attr:doc-14', { approved: true }); + await probe.release(); + }, + expect: { + status: 'completed', + output: 'probed:doc-14/approved', + }, +}; diff --git a/workbench/sim-world/scenarios/cancel-mid-step.ts b/workbench/sim-world/scenarios/cancel-mid-step.ts new file mode 100644 index 0000000000..6df51ca737 --- /dev/null +++ b/workbench/sim-world/scenarios/cancel-mid-step.ts @@ -0,0 +1,19 @@ +import type { ScenarioSpec } from '@workflow/world-sim'; + +export const scenario: ScenarioSpec = { + id: 'cancel-mid-step', + name: 'cancellation lands mid-step', + description: + 'The run is cancelled inside the step_started commit, so the step body ' + + 'runs against an already-terminal run and its step_completed is the only ' + + 'write the world still accepts.', + workflow: 'approvalWorkflow', + input: ['doc-5'], + script: async (sim) => { + const wf = sim.writer.orchestrator(); + await wf.runToEventCommitted('step_started', 'reserveInventory'); + await sim.cancelRun('operator pulled the plug'); + await wf.release(); + }, + expect: { status: 'cancelled' }, +}; diff --git a/workbench/sim-world/scenarios/claimed-payload-under-fork.ts b/workbench/sim-world/scenarios/claimed-payload-under-fork.ts new file mode 100644 index 0000000000..22a014ce59 --- /dev/null +++ b/workbench/sim-world/scenarios/claimed-payload-under-fork.ts @@ -0,0 +1,72 @@ +import type { ScenarioSpec } from '@workflow/world-sim'; + +export const scenario: ScenarioSpec = { + id: 'claimed-payload-under-fork', + name: 'the same payload, with a branch waiting for it', + description: + 'The control for the scenario above: same movements, same tempo, same ' + + 'three deliveries in one resume, same log order β€” but one branch awaits ' + + 'the hook, so the payload arrives claimed. Its barrier registers armed, ' + + 'the wait no longer parks behind an entry that cannot resolve itself, ' + + 'and the step result gates on the wait the ordinary way. The same ' + + 'assertion is made here, and it holds: whatever the other scenario ' + + 'shows, it is not caused by the payload existing.', + workflow: 'claimedPayloadForkWorkflow', + input: ['doc-33'], + script: async (sim) => { + const wf = sim.writer.orchestrator(); + const body = sim.writer.step('pokedWork'); + + // Movement for movement the same as `unclaimed-payload-under-fork`; see + // that file for why each hold is where it is. The single difference is + // the workflow under it, which has a branch awaiting the hook. + await wf.runToEventCommitted('wait_created'); + await sim.deliverHook('poke:doc-33', { kind: 'poke' }); + + const atBody = body.runToEventProduced('step_completed'); + await wf.release(); + await atBody; + + const atWait = wf.runToEventCommitted('wait_completed'); + const fired = sim.deliverQueued( + (pending) => + pending.find((m) => m.readyAtMs > sim.world.nowMs())?.messageId + ); + await atWait; + + const atCommitted = body.runToEventCommitted('step_completed'); + await body.release(); + await atCommitted; + await body.release(); + + await wf.release(); + sim.check('the watchdog fired while the step result was held', await fired); + + const events = sim.world.events(); + const correlationOf = (shortName: string) => + sim.world.steps().find((s) => s.stepName.endsWith(shortName))?.stepId; + const at = (eventType: string, shortName: string) => { + const correlationId = correlationOf(shortName); + return events.findIndex( + (e) => e.eventType === eventType && e.correlationId === correlationId + ); + }; + const waitResolved = events.findIndex( + (e) => e.eventType === 'wait_completed' + ); + const stepResolved = at('step_completed', 'pokedWork'); + const sleepBranchResumed = at('step_created', 'afterPokedSleep'); + const stepBranchResumed = at('step_created', 'afterPokedStep'); + + const waitWasResolvedFirst = waitResolved < stepResolved; + const sleepBranchResumedFirst = sleepBranchResumed < stepBranchResumed; + sim.check( + 'the branch resolved first by the log is the branch that resumes first', + waitWasResolvedFirst === sleepBranchResumedFirst + ); + }, + expect: { + status: 'completed', + output: 'afterStep:doc-33|afterSleep:doc-33', + }, +}; diff --git a/workbench/sim-world/scenarios/count-hook-after-timeout.ts b/workbench/sim-world/scenarios/count-hook-after-timeout.ts new file mode 100644 index 0000000000..0993d148c1 --- /dev/null +++ b/workbench/sim-world/scenarios/count-hook-after-timeout.ts @@ -0,0 +1,19 @@ +import type { ScenarioSpec } from '@workflow/world-sim'; + +export const scenario: ScenarioSpec = { + id: 'count-hook-after-timeout', + name: 'count: hook lands AFTER the timeout, branches differ by step count', + description: + 'The shape PR #3147 identified as the amplifier: settle emits one step, ' + + 'recovery emits two, so flipping the branch on replay renames every ' + + 'correlation ID after the fork.', + workflow: 'stepCountForkWorkflow', + input: ['doc-21'], + script: async (sim) => { + const wf = sim.writer.orchestrator(); + await wf.runToEventCommitted('wait_completed'); + await sim.deliverHook('count:doc-21', { approved: true }); + await wf.release(); + }, + expect: { status: 'completed' }, +}; diff --git a/workbench/sim-world/scenarios/count-hook-before-timeout.ts b/workbench/sim-world/scenarios/count-hook-before-timeout.ts new file mode 100644 index 0000000000..732ca3a47f --- /dev/null +++ b/workbench/sim-world/scenarios/count-hook-before-timeout.ts @@ -0,0 +1,15 @@ +import type { ScenarioSpec } from '@workflow/world-sim'; + +export const scenario: ScenarioSpec = { + id: 'count-hook-before-timeout', + name: 'count: hook lands BEFORE the timeout, branches differ by step count', + workflow: 'stepCountForkWorkflow', + input: ['doc-22'], + script: async (sim) => { + const wf = sim.writer.orchestrator(); + await wf.runToEventProduced('wait_completed'); + await sim.deliverHook('count:doc-22', { approved: true }); + await wf.release(); + }, + expect: { status: 'completed' }, +}; diff --git a/workbench/sim-world/scenarios/deadline-expires.ts b/workbench/sim-world/scenarios/deadline-expires.ts new file mode 100644 index 0000000000..cac60a74c0 --- /dev/null +++ b/workbench/sim-world/scenarios/deadline-expires.ts @@ -0,0 +1,12 @@ +import type { ScenarioSpec } from '@workflow/world-sim'; + +export const scenario: ScenarioSpec = { + id: 'deadline-expires', + name: 'deadline expires with no hook', + description: + 'Nothing delivers the hook, so the run finishes on the timer. One hour ' + + 'of virtual time, no wall-clock wait.', + workflow: 'approvalWithDeadlineWorkflow', + input: ['doc-3', '1h'], + expect: { status: 'completed', output: 'timed-out' }, +}; diff --git a/workbench/sim-world/scenarios/deadline-hook-wins.ts b/workbench/sim-world/scenarios/deadline-hook-wins.ts new file mode 100644 index 0000000000..f39c7cb40d --- /dev/null +++ b/workbench/sim-world/scenarios/deadline-hook-wins.ts @@ -0,0 +1,16 @@ +import type { ScenarioSpec } from '@workflow/world-sim'; + +export const scenario: ScenarioSpec = { + id: 'deadline-hook-wins', + name: 'hook beats its deadline', + workflow: 'approvalWithDeadlineWorkflow', + input: ['doc-2', '1h'], + script: async (sim) => { + const wf = sim.writer.orchestrator(); + // Hold the orchestrator the moment the deadline timer becomes durable. + await wf.runToEventCommitted('wait_created'); + await sim.deliverHook('approval:doc-2', { approved: true }); + await wf.release(); + }, + expect: { status: 'completed', output: 'approved' }, +}; diff --git a/workbench/sim-world/scenarios/fence-catches-benign-direction.ts b/workbench/sim-world/scenarios/fence-catches-benign-direction.ts new file mode 100644 index 0000000000..acd947d9d7 --- /dev/null +++ b/workbench/sim-world/scenarios/fence-catches-benign-direction.ts @@ -0,0 +1,43 @@ +import type { ScenarioSpec } from '@workflow/world-sim'; + +export const scenario: ScenarioSpec = { + id: 'fence-catches-benign-direction', + name: 'fence: the guard catches the harmless direction, not the harmful one', + description: + 'The control that shows the fence is not merely weak here β€” it is aimed ' + + 'the wrong way. Same two racing steps, fence on, completions 5ms apart so ' + + 'no millisecond tie is in play. This time the withheld completion is the ' + + 'log-LATER one (`fast`), so the reader is behind the tail rather than ' + + 'holding a hole: snapshot(t=0) < marker(t=5), and the inline `step_started` ' + + 'claim for the branch step IS rejected β€” twice β€” forcing a reload that ' + + 're-decides on the full log. Exactly the self-correction one would expect ' + + 'the guard to provide. But this direction never needed it: a reader that ' + + 'sees only `slow` β€” the log-FIRST completion β€” already agrees with the ' + + 'log about who won. Flip which one is hidden (the scenario ' + + 'above) and the fence goes silent on the case that actually corrupts. The ' + + 'watermark covers the benign half of the race and misses the dangerous ' + + 'half β€” a hole in the middle of the log moves no high-water mark. That ' + + 'asymmetry is the whole argument for the second half of the fence, the ' + + 'count, which asks a question the mark cannot: not "how new is your ' + + 'newest?" but "how many do you hold below it?".', + workflow: 'stepVsStepForkWorkflow', + input: ['doc-28'], + preconditionGuard: true, + script: async (sim) => { + const fast = sim.writer.step('fast'); + const slow = sim.writer.step('slow'); + const atFast = fast.runToEventProduced('step_completed'); + const atSlow = slow.runToEventProduced('step_completed'); + await atFast; + await atSlow; + // `slow` commits visibly at t=0; the hole is armed against `fast`, which + // commits at t=5 and is therefore the newest out-of-band write. + await slow.release(); + sim.advanceTime(5); + sim.withholdNextEvent(1); + await fast.release(); + }, + // No violation: the fence fires, the run reloads and takes `afterSlow`, + // which is what the log says. Contrast the scenario above, same fence. + expect: { status: 'completed', output: 'afterSlow:doc-28' }, +}; diff --git a/workbench/sim-world/scenarios/fork-hook-after-timeout.ts b/workbench/sim-world/scenarios/fork-hook-after-timeout.ts new file mode 100644 index 0000000000..82c416075e --- /dev/null +++ b/workbench/sim-world/scenarios/fork-hook-after-timeout.ts @@ -0,0 +1,21 @@ +import type { ScenarioSpec } from '@workflow/world-sim'; + +export const scenario: ScenarioSpec = { + id: 'fork-hook-after-timeout', + name: 'fork: hook arrives just AFTER the timeout, before the branch commits', + description: + 'The timeout wins the race, so the first execution takes step 3 β€” but ' + + 'the payload is committed before that branch reaches the log. A replay ' + + 'reaching the race sees both competitors resolvable.', + workflow: 'hookTimeoutForkWorkflow', + input: ['doc-17'], + script: async (sim) => { + const wf = sim.writer.orchestrator(); + await wf.runToEventCommitted('wait_completed'); + await sim.deliverHook('fork:doc-17', { approved: true }); + await wf.release(); + }, + // Deliberately unpinned: which branch the first execution takes is the + // question. The replay check is the judge of whether it is reproducible. + expect: { status: 'completed' }, +}; diff --git a/workbench/sim-world/scenarios/fork-hook-before-timeout.ts b/workbench/sim-world/scenarios/fork-hook-before-timeout.ts new file mode 100644 index 0000000000..a6a9fb2209 --- /dev/null +++ b/workbench/sim-world/scenarios/fork-hook-before-timeout.ts @@ -0,0 +1,19 @@ +import type { ScenarioSpec } from '@workflow/world-sim'; + +export const scenario: ScenarioSpec = { + id: 'fork-hook-before-timeout', + name: 'fork: hook arrives just BEFORE the timeout commits', + description: + 'The mirror of the case above, one world call earlier: hook_received now ' + + 'precedes wait_completed in the log, so log position should hand the race ' + + 'to the hook and fork the other way.', + workflow: 'hookTimeoutForkWorkflow', + input: ['doc-20'], + script: async (sim) => { + const wf = sim.writer.orchestrator(); + await wf.runToEventProduced('wait_completed'); + await sim.deliverHook('fork:doc-20', { approved: true }); + await wf.release(); + }, + expect: { status: 'completed' }, +}; diff --git a/workbench/sim-world/scenarios/fork-hook-wins.ts b/workbench/sim-world/scenarios/fork-hook-wins.ts new file mode 100644 index 0000000000..bae67117a9 --- /dev/null +++ b/workbench/sim-world/scenarios/fork-hook-wins.ts @@ -0,0 +1,15 @@ +import type { ScenarioSpec } from '@workflow/world-sim'; + +export const scenario: ScenarioSpec = { + id: 'fork-hook-wins', + name: 'fork: hook arrives before the timeout', + workflow: 'hookTimeoutForkWorkflow', + input: ['doc-18'], + script: async (sim) => { + const wf = sim.writer.orchestrator(); + await wf.runToEventCommitted('wait_created'); + await sim.deliverHook('fork:doc-18', { approved: true }); + await wf.release(); + }, + expect: { status: 'completed', output: 'step2:doc-18' }, +}; diff --git a/workbench/sim-world/scenarios/fork-timeout-wins.ts b/workbench/sim-world/scenarios/fork-timeout-wins.ts new file mode 100644 index 0000000000..a3ee4d28fd --- /dev/null +++ b/workbench/sim-world/scenarios/fork-timeout-wins.ts @@ -0,0 +1,9 @@ +import type { ScenarioSpec } from '@workflow/world-sim'; + +export const scenario: ScenarioSpec = { + id: 'fork-timeout-wins', + name: 'fork: hook never arrives, timeout decides', + workflow: 'hookTimeoutForkWorkflow', + input: ['doc-19'], + expect: { status: 'completed', output: 'step3:doc-19' }, +}; diff --git a/workbench/sim-world/scenarios/hook-at-hook-created.ts b/workbench/sim-world/scenarios/hook-at-hook-created.ts new file mode 100644 index 0000000000..8e2ea10fdf --- /dev/null +++ b/workbench/sim-world/scenarios/hook-at-hook-created.ts @@ -0,0 +1,26 @@ +import type { ScenarioSpec } from '@workflow/world-sim'; + +export const scenario: ScenarioSpec = { + id: 'hook-at-hook-created', + name: 'hook arrives the instant it is registered', + description: + 'Delivered inside the hook_created commit, before the workflow has even ' + + 'been resumed to schedule the step it runs in parallel with.', + workflow: 'approvalWorkflow', + input: ['doc-1'], + script: async (sim) => { + const wf = sim.writer.orchestrator(); + await wf.runToEventCommitted('hook_created', { + token: 'approval:doc-1', + }); + await sim.deliverHook('approval:doc-1', { + approved: false, + reviewer: 'grace', + }); + await wf.release(); + }, + expect: { + status: 'completed', + output: { status: 'released:reserved:doc-1', reviewer: 'grace' }, + }, +}; diff --git a/workbench/sim-world/scenarios/hook-at-step-completed.ts b/workbench/sim-world/scenarios/hook-at-step-completed.ts new file mode 100644 index 0000000000..5d9edbc431 --- /dev/null +++ b/workbench/sim-world/scenarios/hook-at-step-completed.ts @@ -0,0 +1,26 @@ +import type { ScenarioSpec } from '@workflow/world-sim'; + +export const scenario: ScenarioSpec = { + id: 'hook-at-step-completed', + name: 'hook arrives inside the step_completed commit', + description: + 'Same workflow and same result, but hook_received now lands after ' + + 'step_completed. Diff this event stream against the previous scenario. ' + + 'Note the writer: the result is committed by the step body, not by the ' + + 'orchestrator.', + workflow: 'approvalWorkflow', + input: ['doc-1'], + script: async (sim) => { + const reserve = sim.writer.step('reserveInventory'); + await reserve.runToEventCommitted('step_completed'); + await sim.deliverHook('approval:doc-1', { + approved: true, + reviewer: 'ada', + }); + await reserve.release(); + }, + expect: { + status: 'completed', + output: { status: 'settled:reserved:doc-1', reviewer: 'ada' }, + }, +}; diff --git a/workbench/sim-world/scenarios/hook-at-step-started.ts b/workbench/sim-world/scenarios/hook-at-step-started.ts new file mode 100644 index 0000000000..14054d7cd3 --- /dev/null +++ b/workbench/sim-world/scenarios/hook-at-step-started.ts @@ -0,0 +1,24 @@ +import type { ScenarioSpec } from '@workflow/world-sim'; + +export const scenario: ScenarioSpec = { + id: 'hook-at-step-started', + name: 'hook arrives inside the step_started commit', + description: + 'The hook payload is written after step_started is durable and before ' + + 'the workflow is resumed, so hook_received precedes step_completed in the log.', + workflow: 'approvalWorkflow', + input: ['doc-1'], + script: async (sim) => { + const wf = sim.writer.orchestrator(); + await wf.runToEventCommitted('step_started', 'reserveInventory'); + await sim.deliverHook('approval:doc-1', { + approved: true, + reviewer: 'ada', + }); + await wf.release(); + }, + expect: { + status: 'completed', + output: { status: 'settled:reserved:doc-1', reviewer: 'ada' }, + }, +}; diff --git a/workbench/sim-world/scenarios/hook-never-arrives.ts b/workbench/sim-world/scenarios/hook-never-arrives.ts new file mode 100644 index 0000000000..02bc2c4ae7 --- /dev/null +++ b/workbench/sim-world/scenarios/hook-never-arrives.ts @@ -0,0 +1,12 @@ +import type { ScenarioSpec } from '@workflow/world-sim'; + +export const scenario: ScenarioSpec = { + id: 'hook-never-arrives', + name: 'a hook that never arrives stalls instead of hanging', + description: + 'The expected outcome is a stall: the queue drains, the run is still ' + + 'running, and the report names the hook nobody delivered.', + workflow: 'blockedOnHookWorkflow', + input: ['doc-4'], + expect: { status: 'stalled' }, +}; diff --git a/workbench/sim-world/scenarios/hook-on-execution-state.ts b/workbench/sim-world/scenarios/hook-on-execution-state.ts new file mode 100644 index 0000000000..723e39799d --- /dev/null +++ b/workbench/sim-world/scenarios/hook-on-execution-state.ts @@ -0,0 +1,27 @@ +import type { ScenarioSpec } from '@workflow/world-sim'; + +export const scenario: ScenarioSpec = { + id: 'hook-on-execution-state', + name: 'hook is delivered once execution state says both steps are done', + description: + 'The wait is on world state rather than on one named point: stop ' + + 'whichever step body commits the completion that takes the count to two, ' + + 'then deliver. The payload is buffered before the workflow ever awaits ' + + 'the hook. A `where` predicate cannot be re-checked against history, so ' + + 'this one wait is edge-triggered and leans on its own timeout.', + workflow: 'stagedApprovalWorkflow', + input: ['doc-6'], + script: async (sim) => { + const anyStep = sim.writer.anyStep(); + await anyStep.runToEventCommitted('step_completed', { + where: (world) => + world.steps().filter((s) => s.status === 'completed').length === 2, + }); + await sim.deliverHook('approval:doc-6', { approved: true }); + await anyStep.release(); + }, + expect: { + status: 'completed', + output: 'settled:reserved:doc-6/confirmed', + }, +}; diff --git a/workbench/sim-world/scenarios/in-flight-after-decision.ts b/workbench/sim-world/scenarios/in-flight-after-decision.ts new file mode 100644 index 0000000000..fd8a63f7d3 --- /dev/null +++ b/workbench/sim-world/scenarios/in-flight-after-decision.ts @@ -0,0 +1,69 @@ +import type { ScenarioSpec } from '@workflow/world-sim'; + +export const scenario: ScenarioSpec = { + id: 'in-flight-after-decision', + name: 'in-flight: A commits AFTER the decision β€” no guard can see it', + description: + 'The residual, and the reason the append-tail fence noted in ' + + "workflow-server's `lib/ulid.ts` is still open. Same log order " + + '(A=hook_received, B=wait_completed) and the same decision on a log ' + + 'missing the hook, but this time the receiver commits after C rather than ' + + 'before it: visibility is (B, C, A). Both halves of the fence are armed ' + + 'and neither fires, and neither could β€” a check is part of the write it ' + + 'guards, evaluated against the log as it stands at that instant, so it can ' + + 'only compare against events that already exist. At every point where a ' + + 'write of this run is checked, the hook does not exist. Then it appears, ' + + 'behind everything, and the log says the hook beat a timeout the run ' + + 'resolved the other way. ' + + 'Getting there needs the run to be quiescent when the hook lands, because ' + + 'the count guard catches this same hole on whatever the run writes NEXT β€” ' + + 'late, after the wrong branch has already run, which is a different and ' + + 'much worse outcome than catching it in time. So the hook is released ' + + 'while the orchestrator is held inside `wait_created`, the last write of ' + + 'the delivery: the run then sleeps, and the next delivery cold-starts on a ' + + 'log it can no longer follow. Detectability is inversely related to how ' + + 'late the write commits, which is the opposite of the intuition that a ' + + 'slower write is more dangerous the longer it takes. ' + + 'This is the one an append-only log closes outright, and it closes it by ' + + 'construction rather than by catching anything: the append-tail fence is ' + + 'unnecessary when the tail is the only place a write can land. The late ' + + 'hook sorts last, the next delivery replays a log it can follow, and no ' + + 'guard has to fire.', + workflow: 'lateAppendForkWorkflow', + input: ['doc-31'], + preconditionGuard: true, + countGuard: true, + script: async (sim) => { + const wf = sim.writer.orchestrator(); + await wf.runToEventProduced('wait_completed'); + const hook = await sim.beginHookDelivery('count:doc-31', { + approved: true, + }); + + // Let the delivery play out on the branch the visible log implied, and + // catch it inside the `wait_created` that ends it. That write is already + // durable; nothing of this run will be checked again until the timer + // fires. + await wf.runToEventCommitted('wait_created'); + sim.check( + 'nothing was fenced β€” every write so far passed both guards', + sim.world.rejections().length === 0 + ); + + await hook.commit(); + await wf.release(); + }, + // FAILS TODAY, and worse than the others: the corruption is not merely + // latent. The next delivery replays a log that says the hook won, finds + // `settle` where `recoverFirst` belongs, and gives up after its recovery + // replays β€” so the run dies rather than completing wrongly. Of the six reds, + // this is the one with no known fix: both guards are on, and closing it needs + // the append-tail fence that does not exist yet. + // + // "Completes" is the whole assertion, and here it is not a formality: this is + // the one red where the run does not reach a terminal success at all. Under + // the flag it does, and the log replays β€” same sentence, other answer. + expect: { + status: 'completed', + }, +}; diff --git a/workbench/sim-world/scenarios/in-flight-before-decision-counted.ts b/workbench/sim-world/scenarios/in-flight-before-decision-counted.ts new file mode 100644 index 0000000000..7caea5f29c --- /dev/null +++ b/workbench/sim-world/scenarios/in-flight-before-decision-counted.ts @@ -0,0 +1,67 @@ +import type { ScenarioSpec } from '@workflow/world-sim'; + +export const scenario: ScenarioSpec = { + id: 'in-flight-before-decision-counted', + name: 'in-flight: same tempo, count guard ON β€” the write is fenced', + description: + 'Identical to the scenario above with the count half of the fence armed: ' + + 'the caller sends how many events it had loaded at or below its own ' + + 'watermark, and the world compares that against how many the log actually ' + + 'holds there. The hook committed in the meantime, so the log holds one ' + + 'more than the caller loaded and C is rejected with a 412 β€” even though ' + + 'the watermark comparison passes. The orchestrator reloads, sees the hook ' + + 'ahead of the timeout in log order, re-decides the fork as "arrived", and ' + + 'commits the branch the log agrees with. This is the regression test for ' + + 'the half of the fence a high-water mark cannot express: same fault, same ' + + 'tempo, one flag apart. This is also the production-shaped one of the ' + + 'pair: since #3145 `@workflow/core` sends the count on every ' + + 'replay-context create and the server checks it by default, so the flag ' + + 'below is now the default rather than an opt-in, and it is the twin above ' + + 'that has to switch it off. ' + + 'Under an append-only log the 412 still fires and now saves nothing: the ' + + 'count is taken at or below the caller’s watermark, and the watermark ' + + 'is a millisecond, so a hook that commits after the timeout inside the ' + + 'same virtual millisecond still counts as "at or below" it. The reload ' + + 'sees the hook behind the timeout in log order, re-decides the same way, ' + + 'and settles β€” a restart with nothing to correct. That is what the ' + + 'fence costs once the log is append-only: false positives at millisecond ' + + 'granularity, in exchange for a hole that can no longer open.', + workflow: 'stepCountForkWorkflow', + input: ['doc-30'], + preconditionGuard: true, + countGuard: true, + script: async (sim) => { + const wf = sim.writer.orchestrator(); + await wf.runToEventProduced('wait_completed'); + const hook = await sim.beginHookDelivery('count:doc-30', { + approved: true, + }); + await wf.runToEventProduced('step_started'); + await hook.commit(); + await wf.release(); + + // The point of the scenario, and the part that is true in both worlds: the + // count half of the fence fires where the watermark half did not. What the + // reload then decides is a different question and belongs to the world β€” + // mint-ordered it corrects the branch, append-only it re-confirms it β€” so + // that half is left to the trace. Asserting it here is what forced this + // scenario to carry two expectations, and it was never what distinguished + // it from its uncounted twin. + // Matched on the count half's own message, not on "something was + // rejected". The twin rejects too β€” its writes hit `RunExpiredError` once + // the corrupted branch has run β€” so a bare `rejections().length > 0` would + // hold there as well and assert nothing about the guard. + sim.check( + 'the count guard fenced the write the watermark let through', + sim.world + .rejections() + .some((r) => r.message.includes('at or below the caller')) + ); + }, + // The rejection and the reload show up in the trace as `!!` lines. Whichever + // branch the reload lands on, it is the one the durable log implies β€” so + // there is nothing to diverge, in either world. + expect: { + status: 'completed', + }, +}; diff --git a/workbench/sim-world/scenarios/in-flight-before-decision.ts b/workbench/sim-world/scenarios/in-flight-before-decision.ts new file mode 100644 index 0000000000..47759e8441 --- /dev/null +++ b/workbench/sim-world/scenarios/in-flight-before-decision.ts @@ -0,0 +1,86 @@ +import type { ScenarioSpec } from '@workflow/world-sim'; + +export const scenario: ScenarioSpec = { + id: 'in-flight-before-decision', + name: 'in-flight: A commits BEFORE the decision is written β€” count guard off', + description: + 'Log order is (A=hook_received, B=wait_completed); visibility is ' + + '(B, A, C). The webhook receiver has entered its handler and minted the ' + + "hook's event id, so position A is spoken for, but the write has not " + + 'landed. The orchestrator then commits the timeout at B β€” behind a ' + + 'position it cannot see β€” reads a log that genuinely does not contain the ' + + 'hook, and takes the settle branch. Nothing is withheld from any read. ' + + 'The receiver commits while the orchestrator is held at the produced ' + + 'point of C, so by the time C is checked the hole has closed and the log ' + + 'holds an event the writer never loaded. The watermark guard is on and ' + + 'passes anyway, by construction: the marker moves to the ULID time of the ' + + "hook, which sorts at or below the writer's own snapshot, so " + + '`stateUpdatedAt < marker` is false. It corrupts β€” the same corruption as ' + + 'the doc-23 pair, reached without a stale read. ' + + 'Under an append-only log there is no position to be spoken for: the hook ' + + 'commits after the timeout and therefore sorts after it, the log says the ' + + 'timeout won, and the settle branch the run took is the one the log ' + + 'describes. Same tempo, same expectation, no corruption. Which branch the ' + + 'run ends on is decided by what its reads returned, and what a read ' + + 'returns is the one thing the flag changes β€” so the branch is reported, ' + + 'not asserted. What is asserted holds in both worlds: the run completes, ' + + 'and the log it wrote replays back into the run that wrote it.', + workflow: 'stepCountForkWorkflow', + input: ['doc-29'], + preconditionGuard: true, + // Explicit against the default, which follows the fence: this scenario is the + // watermark half on its own, and the `-counted` twin below is the same tempo + // with production's second half restored. One flag apart is the whole point. + countGuard: false, + script: async (sim) => { + const wf = sim.writer.orchestrator(); + + // Stop the orchestrator before it submits the timeout, so the receiver + // gets the earlier position. `produced` is the pre-submit point: nothing + // has been minted for `wait_completed` yet. + await wf.runToEventProduced('wait_completed'); + + const hook = await sim.beginHookDelivery('count:doc-29', { + approved: true, + }); + // The condition is the same in both worlds β€” the hook is not in the log β€” + // but what that *means* is not, and the trace should not claim otherwise. + // Mint-ordered, the reserved position is binding and the hook already owns + // a slot ahead of everything the orchestrator is about to write; under an + // append-only log the reservation decides nothing, and the hook is simply + // absent until it lands. + sim.check( + sim.appendOnlyLog + ? 'the hook has not landed; where it lands is not decided yet' + : 'the hook owns a log position but is nowhere in the log', + sim.world.events().every((e) => e.eventType !== 'hook_received') + ); + + // B commits behind A, then the orchestrator decides the fork on a log + // that has a hole in it. Hold it before that decision is submitted. The + // claim for a branch step is one `events.create` carrying `step_started` + // β€” the `step_created` ahead of it is appended by the same write β€” so + // `step_started` is the call point the decision passes through. + const decision = await wf.runToEventProduced('step_started'); + sim.check( + 'the live pass decided the fork without the hook', + JSON.stringify(decision.ctx.request?.eventData).includes('settle') + ); + + // A lands, behind the snapshot C was decided on. + await hook.commit(); + await wf.release(); + }, + // FAILS TODAY, like the stale-read scenarios above, and needs no stale read + // to do it. Note what is *not* here: the settle/recover branch. The run + // settles in both worlds β€” that part is not the bug. The bug is that + // mint-ordered, the log it left behind says the hook came first, so a cold + // replay of that log takes `recoverFirst` and diverges from the run that + // wrote it. That divergence is the failure, it is what `verifyReplay` + // catches, and it is stated the same way in both worlds. The fix is known + // and one flag away β€” see the scenario below, this one with the count guard + // on and green. + expect: { + status: 'completed', + }, +}; diff --git a/workbench/sim-world/scenarios/index.ts b/workbench/sim-world/scenarios/index.ts new file mode 100644 index 0000000000..a265a957c0 --- /dev/null +++ b/workbench/sim-world/scenarios/index.ts @@ -0,0 +1,188 @@ +/** + * The scenario book: one file per scenario, and this index. + * + * Each entry is one workflow plus a script saying how the run's writers + * interleave and when external input arrives. Pairs of scenarios that differ + * only in *when* a hook is delivered are the interesting ones β€” same workflow, + * same input, same result, different event log β€” because that difference is + * exactly what a real deployment leaves to chance. + * + * How to write one β€” the three moves, which writer commits which event, what + * to assert and what not to β€” is in the workbench README. Start by copying the + * file next door. + * + * @see ../README.md#adding-a-scenario + */ + +import type { ScenarioSpec } from '@workflow/world-sim'; +import { scenario as attrFromStepBody } from './attr-from-step-body.ts'; +import { scenario as attrHookAfterStep } from './attr-hook-after-step.ts'; +import { scenario as attrHookBeforeStep } from './attr-hook-before-step.ts'; +import { scenario as cancelMidStep } from './cancel-mid-step.ts'; +import { scenario as claimedPayloadUnderFork } from './claimed-payload-under-fork.ts'; +import { scenario as countHookAfterTimeout } from './count-hook-after-timeout.ts'; +import { scenario as countHookBeforeTimeout } from './count-hook-before-timeout.ts'; +import { scenario as deadlineExpires } from './deadline-expires.ts'; +import { scenario as deadlineHookWins } from './deadline-hook-wins.ts'; +import { scenario as fenceCatchesBenignDirection } from './fence-catches-benign-direction.ts'; +import { scenario as forkHookAfterTimeout } from './fork-hook-after-timeout.ts'; +import { scenario as forkHookBeforeTimeout } from './fork-hook-before-timeout.ts'; +import { scenario as forkHookWins } from './fork-hook-wins.ts'; +import { scenario as forkTimeoutWins } from './fork-timeout-wins.ts'; +import { scenario as hookAtHookCreated } from './hook-at-hook-created.ts'; +import { scenario as hookAtStepCompleted } from './hook-at-step-completed.ts'; +import { scenario as hookAtStepStarted } from './hook-at-step-started.ts'; +import { scenario as hookNeverArrives } from './hook-never-arrives.ts'; +import { scenario as hookOnExecutionState } from './hook-on-execution-state.ts'; +import { scenario as inFlightAfterDecision } from './in-flight-after-decision.ts'; +import { scenario as inFlightBeforeDecision } from './in-flight-before-decision.ts'; +import { scenario as inFlightBeforeDecisionCounted } from './in-flight-before-decision-counted.ts'; +import { scenario as longSleep } from './long-sleep.ts'; +import { scenario as parallelSteps } from './parallel-steps.ts'; +import { scenario as peekHookAfterBranch } from './peek-hook-after-branch.ts'; +import { scenario as peekHookAtRegistration } from './peek-hook-at-registration.ts'; +import { scenario as peekHookBeforeBranch } from './peek-hook-before-branch.ts'; +import { scenario as raceDuplicateDelivery } from './race-duplicate-delivery.ts'; +import { scenario as raceHookAfterProbe } from './race-hook-after-probe.ts'; +import { scenario as raceHookBeforeProbe } from './race-hook-before-probe.ts'; +import { scenario as smokeNoSteps } from './smoke-no-steps.ts'; +import { scenario as smokeOneStep } from './smoke-one-step.ts'; +import { scenario as staleReadEqualStepCounts } from './stale-read-equal-step-counts.ts'; +import { scenario as staleReadStepCountFork } from './stale-read-step-count-fork.ts'; +import { scenario as staleReadStepCountForkFenced } from './stale-read-step-count-fork-fenced.ts'; +import { scenario as stepRetriesTwice } from './step-retries-twice.ts'; +import { scenario as stepVsStepFork } from './step-vs-step-fork.ts'; +import { scenario as stepVsStepForkFenced } from './step-vs-step-fork-fenced.ts'; +import { scenario as unclaimedPayloadUnderFork } from './unclaimed-payload-under-fork.ts'; +import { scenario as writersIndependentStepBodies } from './writers-independent-step-bodies.ts'; +import { scenario as writersScriptedTempo } from './writers-scripted-tempo.ts'; + +/** + * The book, in reading order. Order is the only thing this file decides: + * simplest first, then each pair of near-identical scenarios adjacent, so a + * reader meets a distinction right after the thing it is a distinction from. + */ +export const scenarios: ScenarioSpec[] = [ + // ------------------------------------------------------------------------- + // Smoke: the smallest logs there are. If the replay check cannot agree with + // itself here, the problem is the check, not the workflow. + // ------------------------------------------------------------------------- + smokeNoSteps, + smokeOneStep, + + // ------------------------------------------------------------------------- + // The three placements of one hook, relative to one step. + // ------------------------------------------------------------------------- + hookAtStepStarted, + hookAtStepCompleted, + hookAtHookCreated, + + // ------------------------------------------------------------------------- + // Racing a hook against a timer β€” both branches, on demand. + // ------------------------------------------------------------------------- + deadlineHookWins, + deadlineExpires, + + // ------------------------------------------------------------------------- + // Termination properties. + // ------------------------------------------------------------------------- + longSleep, + hookNeverArrives, + + // ------------------------------------------------------------------------- + // Step lifecycle. + // ------------------------------------------------------------------------- + stepRetriesTwice, + parallelSteps, + hookOnExecutionState, + + // ------------------------------------------------------------------------- + // A hook peek: branching on *when* a payload arrived. + // + // Same workflow, same input, two deliveries one world call apart. The only + // difference is whether hook_received lands before or after the branch's own + // events β€” which is exactly the thing a replay cannot re-derive. + // ------------------------------------------------------------------------- + peekHookBeforeBranch, + peekHookAfterBranch, + peekHookAtRegistration, + raceHookBeforeProbe, + raceHookAfterProbe, + raceDuplicateDelivery, + + // ------------------------------------------------------------------------- + // Concurrent branches: a hook-gated attribute write racing a step. + // ------------------------------------------------------------------------- + attrHookBeforeStep, + attrHookAfterStep, + attrFromStepBody, + + // ------------------------------------------------------------------------- + // step 1 -> hook-with-timeout -> fork. The payload lands in the window + // between the timeout firing and the chosen branch being committed. + // ------------------------------------------------------------------------- + forkHookAfterTimeout, + forkHookBeforeTimeout, + countHookAfterTimeout, + countHookBeforeTimeout, + staleReadStepCountFork, + staleReadEqualStepCounts, + stepVsStepFork, + stepVsStepForkFenced, + fenceCatchesBenignDirection, + + // ------------------------------------------------------------------------- + // The same fork as the doc-23 pair, but with no stale read anywhere. The + // log's earlier event is simply still IN FLIGHT: its id β€” the log's sort + // key β€” was minted at the handler boundary (workflow-server calls + // `EventId.make()` before it attempts the write, because DynamoDB does not + // generate ids), and the write has not landed. Every reader gets a complete, + // strongly-consistent view of the log; that log just does not contain the + // event yet, and when it finally does the event appears *behind* a position + // readers have already passed. + // + // This is the shape production actually has, now that event-log reads are + // strongly consistent: there is no read to be stale, so `withholdNextEvent` + // models a fault that no longer exists. What differs between the three + // scenarios below is only *when* the in-flight write lands relative to the + // decision it invalidates, and that timing alone decides which guard, if + // any, can see it. + // + // The in-flight writer has to be the out-of-band one. Holding an inline + // step's `step_completed` between mint and commit stalls the orchestrator + // too β€” the runtime awaits every inline step promise before it can decide + // anything β€” so the reader that should misread the log never gets to read + // it. That is not a limitation of the simulator; it is why the hazard needs + // a writer that is not part of the run's own await graph. + // ------------------------------------------------------------------------- + inFlightBeforeDecision, + inFlightBeforeDecisionCounted, + inFlightAfterDecision, + staleReadStepCountForkFenced, + forkHookWins, + forkTimeoutWins, + + // ------------------------------------------------------------------------- + // An unclaimed hook payload under the fork. + // + // The three scenarios above all turn on *when* an event lands. These two + // turn on something the log alone does not show: whether anything in the + // workflow is waiting for it. A payload nobody reads registers a delivery + // barrier that only the runtime's idle net can retire, and every later + // delivery that defers behind hooks parks on it β€” so it changes the order + // other events are delivered in without appearing to do anything at all. + // + // The pair is a controlled comparison: identical steps, identical tempo, + // identical log order, differing only in whether a branch claims the + // payload. + // ------------------------------------------------------------------------- + unclaimedPayloadUnderFork, + claimedPayloadUnderFork, + + // ------------------------------------------------------------------------- + // What the writer vocabulary buys, stated as scenarios. + // ------------------------------------------------------------------------- + writersIndependentStepBodies, + writersScriptedTempo, + cancelMidStep, +]; diff --git a/workbench/sim-world/scenarios/long-sleep.ts b/workbench/sim-world/scenarios/long-sleep.ts new file mode 100644 index 0000000000..a5c7c11226 --- /dev/null +++ b/workbench/sim-world/scenarios/long-sleep.ts @@ -0,0 +1,9 @@ +import type { ScenarioSpec } from '@workflow/world-sim'; + +export const scenario: ScenarioSpec = { + id: 'long-sleep', + name: 'a thirty-day sleep costs nothing', + workflow: 'longSleepWorkflow', + input: ['payload'], + expect: { status: 'completed', output: 'finalized:prepared:payload' }, +}; diff --git a/workbench/sim-world/scenarios/parallel-steps.ts b/workbench/sim-world/scenarios/parallel-steps.ts new file mode 100644 index 0000000000..63061f5271 --- /dev/null +++ b/workbench/sim-world/scenarios/parallel-steps.ts @@ -0,0 +1,9 @@ +import type { ScenarioSpec } from '@workflow/world-sim'; + +export const scenario: ScenarioSpec = { + id: 'parallel-steps', + name: 'two steps suspend together', + workflow: 'parallelStepsWorkflow', + input: ['x'], + expect: { status: 'completed', output: 'prepared:x|finalized:x' }, +}; diff --git a/workbench/sim-world/scenarios/peek-hook-after-branch.ts b/workbench/sim-world/scenarios/peek-hook-after-branch.ts new file mode 100644 index 0000000000..d659bf4f46 --- /dev/null +++ b/workbench/sim-world/scenarios/peek-hook-after-branch.ts @@ -0,0 +1,21 @@ +import type { ScenarioSpec } from '@workflow/world-sim'; + +export const scenario: ScenarioSpec = { + id: 'peek-hook-after-branch', + name: 'peek: hook lands just AFTER the branch step commits', + description: + 'The control. One world call later, so hook_received sits after the ' + + 'branch in the log and a replay reaching the peek still sees nothing.', + workflow: 'hookPeekWorkflow', + input: ['doc-9'], + script: async (sim) => { + const wf = sim.writer.orchestrator(); + await wf.runToEventCommitted('step_started', 'shipWithoutApproval'); + await sim.deliverHook('peek:doc-9', { approved: true }); + await wf.release(); + }, + expect: { + status: 'completed', + output: 'shipped-unapproved:reserved:doc-9', + }, +}; diff --git a/workbench/sim-world/scenarios/peek-hook-at-registration.ts b/workbench/sim-world/scenarios/peek-hook-at-registration.ts new file mode 100644 index 0000000000..0e7c02fcd7 --- /dev/null +++ b/workbench/sim-world/scenarios/peek-hook-at-registration.ts @@ -0,0 +1,18 @@ +import type { ScenarioSpec } from '@workflow/world-sim'; + +export const scenario: ScenarioSpec = { + id: 'peek-hook-at-registration', + name: 'peek: hook lands the instant it is registered', + description: + 'The earliest a payload can possibly arrive. Whichever fork the first ' + + 'execution takes, the replay has to agree with it.', + workflow: 'hookPeekWorkflow', + input: ['doc-10'], + script: async (sim) => { + const wf = sim.writer.orchestrator(); + await wf.runToEventCommitted('hook_created', { token: 'peek:doc-10' }); + await sim.deliverHook('peek:doc-10', { approved: true }); + await wf.release(); + }, + expect: { status: 'completed' }, +}; diff --git a/workbench/sim-world/scenarios/peek-hook-before-branch.ts b/workbench/sim-world/scenarios/peek-hook-before-branch.ts new file mode 100644 index 0000000000..190b5b6323 --- /dev/null +++ b/workbench/sim-world/scenarios/peek-hook-before-branch.ts @@ -0,0 +1,23 @@ +import type { ScenarioSpec } from '@workflow/world-sim'; + +export const scenario: ScenarioSpec = { + id: 'peek-hook-before-branch', + name: 'peek: hook lands just BEFORE the branch step commits', + description: + 'The first execution peeks, sees nothing, and ships unapproved β€” then ' + + 'the payload is committed ahead of that branch in the log. A replay ' + + 'reaching the peek now sees the hook and wants the other fork.', + workflow: 'hookPeekWorkflow', + input: ['doc-8'], + script: async (sim) => { + const wf = sim.writer.orchestrator(); + await wf.runToEventProduced('step_started', 'shipWithoutApproval'); + await sim.deliverHook('peek:doc-8', { approved: true }); + await wf.release(); + }, + // What the first execution actually decided. Anything else is the bug. + expect: { + status: 'completed', + output: 'shipped-unapproved:reserved:doc-8', + }, +}; diff --git a/workbench/sim-world/scenarios/race-duplicate-delivery.ts b/workbench/sim-world/scenarios/race-duplicate-delivery.ts new file mode 100644 index 0000000000..83a2233c21 --- /dev/null +++ b/workbench/sim-world/scenarios/race-duplicate-delivery.ts @@ -0,0 +1,29 @@ +import type { ScenarioSpec } from '@workflow/world-sim'; + +export const scenario: ScenarioSpec = { + id: 'race-duplicate-delivery', + name: 'race: a webhook receiver delivers the same payload twice', + description: + 'Two hook_received events for one hookId, straddling the step result. ' + + 'The consumer is subscribed once; the second payload has nobody to go to. ' + + "Note the arming order: the step body's wait is armed while the " + + 'orchestrator is still held, because releasing first would let the ' + + 'completion slip past.', + workflow: 'hookRaceStepWorkflow', + input: ['doc-13'], + script: async (sim) => { + const wf = sim.writer.orchestrator(); + const probe = sim.writer.step('probe'); + + await wf.runToEventCommitted('step_started', 'probe'); + await sim.deliverHook('race:doc-13', { approved: true }); + + const completion = probe.runToEventCommitted('step_completed'); + await wf.release(); + await completion; + + await sim.deliverHook('race:doc-13', { approved: true }); + await probe.release(); + }, + expect: { status: 'completed' }, +}; diff --git a/workbench/sim-world/scenarios/race-hook-after-probe.ts b/workbench/sim-world/scenarios/race-hook-after-probe.ts new file mode 100644 index 0000000000..e66084adf7 --- /dev/null +++ b/workbench/sim-world/scenarios/race-hook-after-probe.ts @@ -0,0 +1,15 @@ +import type { ScenarioSpec } from '@workflow/world-sim'; + +export const scenario: ScenarioSpec = { + id: 'race-hook-after-probe', + name: 'race: hook lands just AFTER the probe step completes', + workflow: 'hookRaceStepWorkflow', + input: ['doc-12'], + script: async (sim) => { + const probe = sim.writer.step('probe'); + await probe.runToEventCommitted('step_completed'); + await sim.deliverHook('race:doc-12', { approved: true }); + await probe.release(); + }, + expect: { status: 'completed' }, +}; diff --git a/workbench/sim-world/scenarios/race-hook-before-probe.ts b/workbench/sim-world/scenarios/race-hook-before-probe.ts new file mode 100644 index 0000000000..a00364daef --- /dev/null +++ b/workbench/sim-world/scenarios/race-hook-before-probe.ts @@ -0,0 +1,20 @@ +import type { ScenarioSpec } from '@workflow/world-sim'; + +export const scenario: ScenarioSpec = { + id: 'race-hook-before-probe', + name: 'race: hook lands just BEFORE the probe step completes', + description: + 'Both branches of the race are event-log deliveries now, so the winner ' + + 'is decided by delivery-barrier ordering β€” and the script puts ' + + 'hook_received ahead of the step result in the log by holding the step ' + + 'body at the point where it has decided to write and has not yet.', + workflow: 'hookRaceStepWorkflow', + input: ['doc-11'], + script: async (sim) => { + const probe = sim.writer.step('probe'); + await probe.runToEventProduced('step_completed'); + await sim.deliverHook('race:doc-11', { approved: true }); + await probe.release(); + }, + expect: { status: 'completed' }, +}; diff --git a/workbench/sim-world/scenarios/smoke-no-steps.ts b/workbench/sim-world/scenarios/smoke-no-steps.ts new file mode 100644 index 0000000000..4a822f19ab --- /dev/null +++ b/workbench/sim-world/scenarios/smoke-no-steps.ts @@ -0,0 +1,8 @@ +import type { ScenarioSpec } from '@workflow/world-sim'; + +export const scenario: ScenarioSpec = { + id: 'smoke-no-steps', + name: 'smoke: a workflow with no steps at all', + workflow: 'emptyWorkflow', + expect: { status: 'completed', output: 'done' }, +}; diff --git a/workbench/sim-world/scenarios/smoke-one-step.ts b/workbench/sim-world/scenarios/smoke-one-step.ts new file mode 100644 index 0000000000..7e18f4d1c0 --- /dev/null +++ b/workbench/sim-world/scenarios/smoke-one-step.ts @@ -0,0 +1,8 @@ +import type { ScenarioSpec } from '@workflow/world-sim'; + +export const scenario: ScenarioSpec = { + id: 'smoke-one-step', + name: 'smoke: a workflow with one null step', + workflow: 'oneStepWorkflow', + expect: { status: 'completed', output: null }, +}; diff --git a/workbench/sim-world/scenarios/stale-read-equal-step-counts.ts b/workbench/sim-world/scenarios/stale-read-equal-step-counts.ts new file mode 100644 index 0000000000..0dda465206 --- /dev/null +++ b/workbench/sim-world/scenarios/stale-read-equal-step-counts.ts @@ -0,0 +1,25 @@ +import type { ScenarioSpec } from '@workflow/world-sim'; + +export const scenario: ScenarioSpec = { + id: 'stale-read-equal-step-counts', + name: 'corrupt: stale event load with EQUAL step counts (is the amplifier needed?)', + description: + 'Identical fault to the scenario above, but on the fork whose branches ' + + 'each emit exactly one step. If this corrupts too then step-count ' + + 'divergence raises the rate rather than being required.', + workflow: 'hookTimeoutForkWorkflow', + input: ['doc-25'], + script: async (sim) => { + const wf = sim.writer.orchestrator(); + await wf.runToEventProduced('wait_completed'); + sim.withholdNextEvent(1); + await sim.deliverHook('fork:doc-25', { approved: true }); + await wf.release(); + }, + // FAILS TODAY, which answers the question in the name: the amplifier is not + // required for corruption β€” it is required for the *rate* under concurrent + // load, and for divergence deep in a long log. What corruption needs is + // that the flipped branch claim an ordinal the log already gave to a + // differently-named step. + expect: { status: 'completed', output: 'step2:doc-25' }, +}; diff --git a/workbench/sim-world/scenarios/stale-read-step-count-fork-fenced.ts b/workbench/sim-world/scenarios/stale-read-step-count-fork-fenced.ts new file mode 100644 index 0000000000..c738f8f977 --- /dev/null +++ b/workbench/sim-world/scenarios/stale-read-step-count-fork-fenced.ts @@ -0,0 +1,26 @@ +import type { ScenarioSpec } from '@workflow/world-sim'; + +export const scenario: ScenarioSpec = { + id: 'stale-read-step-count-fork-fenced', + name: 'corrupt: same shape, with the optimistic-concurrency fence armed', + description: + 'Identical to the count: fork scenario but with preconditionGuard on, so ' + + 'the World rejects a replay-context write whose stateUpdatedAt snapshot ' + + 'predates the newest out-of-band event. Does the 412 fence stop it? It ' + + 'does: every rejected write is traced as a `!!` line, and the run ' + + 'reconciles instead of diverging.', + workflow: 'stepCountForkWorkflow', + input: ['doc-24'], + preconditionGuard: true, + script: async (sim) => { + const wf = sim.writer.orchestrator(); + await wf.runToEventProduced('wait_completed'); + sim.withholdNextEvent(1); + await sim.deliverHook('count:doc-24', { approved: true }); + await wf.release(); + }, + expect: { + status: 'completed', + output: 'reconciled(recovered:doc-24+second)', + }, +}; diff --git a/workbench/sim-world/scenarios/stale-read-step-count-fork.ts b/workbench/sim-world/scenarios/stale-read-step-count-fork.ts new file mode 100644 index 0000000000..1281d00ee5 --- /dev/null +++ b/workbench/sim-world/scenarios/stale-read-step-count-fork.ts @@ -0,0 +1,31 @@ +import type { ScenarioSpec } from '@workflow/world-sim'; + +export const scenario: ScenarioSpec = { + id: 'stale-read-step-count-fork', + name: 'corrupt: stale event load + step-count fork', + description: + 'All three preconditions from PR #3147 at once. The hook is committed ' + + 'ahead of wait_completed in the log, but withheld from the read the live ' + + 'pass uses β€” so the live pass decides the fork without it, while the ' + + 'durable log says the hook came first. The branches differ by step count, ' + + 'so flipping the fork on replay renames every entity after it.', + workflow: 'stepCountForkWorkflow', + input: ['doc-23'], + script: async (sim) => { + const wf = sim.writer.orchestrator(); + await wf.runToEventProduced('wait_completed'); + // Arm the window, then write behind it: the hook takes log position 7, + // ahead of wait_completed at 8, but the read that decides the fork does + // not see it. + sim.withholdNextEvent(1); + await sim.deliverHook('count:doc-23', { approved: true }); + await wf.release(); + }, + // FAILS TODAY. The log puts the hook ahead of the timeout, so the run that + // agrees with its own log takes the recovery branch. It takes `settle` + // instead and then cannot replay what it wrote. + expect: { + status: 'completed', + output: 'reconciled(recovered:doc-23+second)', + }, +}; diff --git a/workbench/sim-world/scenarios/step-retries-twice.ts b/workbench/sim-world/scenarios/step-retries-twice.ts new file mode 100644 index 0000000000..bd08bb80d5 --- /dev/null +++ b/workbench/sim-world/scenarios/step-retries-twice.ts @@ -0,0 +1,9 @@ +import type { ScenarioSpec } from '@workflow/world-sim'; + +export const scenario: ScenarioSpec = { + id: 'step-retries-twice', + name: 'a step retries twice and then succeeds', + workflow: 'retryingWorkflow', + input: ['charge'], + expect: { status: 'completed', output: 'charge:ok-on-attempt-3' }, +}; diff --git a/workbench/sim-world/scenarios/step-vs-step-fork-fenced.ts b/workbench/sim-world/scenarios/step-vs-step-fork-fenced.ts new file mode 100644 index 0000000000..34b6bfef49 --- /dev/null +++ b/workbench/sim-world/scenarios/step-vs-step-fork-fenced.ts @@ -0,0 +1,60 @@ +import type { ScenarioSpec } from '@workflow/world-sim'; + +export const scenario: ScenarioSpec = { + id: 'step-vs-step-fork-fenced', + name: 'corrupt: two racing STEPS, WITH the precondition fence on', + description: + "Tests the fence's predicate. WorldCapabilities.preconditionGuard is " + + 'documented as rejecting a stale write when a newer OUT-OF-BAND event ' + + '(e.g. a received hook) was recorded. So does it fence a write made ' + + "stale by one of the run's OWN step_completed events? Same fault as the " + + 'scenario above, fence enabled. Verified answer: NO β€” zero ' + + 'PreconditionFailedError rejections, and it corrupts identically. The ' + + 'reason is the shape of the predicate, not the event type: the fence ' + + 'compares the snapshot against a HIGH-WATER MARK of the newest ' + + 'out-of-band write, and rejects only `stateUpdatedAt < marker`. Here the ' + + 'newest such write is the one the reader CAN see (`fast`); the withheld ' + + "one is older, a hole in the middle of the log, so the reader's snapshot " + + 'is never strictly older than the mark. Separating the two completions ' + + 'in virtual time does not change it β€” the miss is structural, not a ' + + 'millisecond-granularity tie. Contrast the hook/wait variant above, ' + + 'where the withheld hook IS the newest out-of-band write and the ' + + 'orchestrator carries a pre-sleep snapshot: strictly older, so the same ' + + 'fence rejects twice and the run self-corrects β€” those rejections show ' + + 'up in the trace as `!!` lines, unasked for. ' + + 'To be precise about which fence: this scenario arms the watermark half ' + + 'ALONE, which is why `countGuard` is switched off below against the ' + + 'default. The count half is aimed at exactly this hole and does catch it, ' + + 'and since #3145 it is armed in production too β€” so what stays red here ' + + 'is the watermark predicate, not a hole anything real still has. See the ' + + 'in-flight trio below, where the two halves are separated and tested one ' + + 'flag apart.', + workflow: 'stepVsStepForkWorkflow', + input: ['doc-27'], + preconditionGuard: true, + // The subject is the watermark predicate on its own. Production arms both + // halves, so the count guard now follows the fence by default; a scenario + // that exists to show what the watermark alone misses has to opt out of it. + countGuard: false, + script: async (sim) => { + const fast = sim.writer.step('fast'); + const slow = sim.writer.step('slow'); + const atFast = fast.runToEventProduced('step_completed'); + const atSlow = slow.runToEventProduced('step_completed'); + await atFast; + await atSlow; + // Who this withheld reader is in production: not this invocation. With + // strongly-consistent reads a single invocation cannot miss its own + // committed write, so the reader that misses one of these two step + // writes is a *concurrent second invocation* of the same run β€” the storm + // shape, which the sim cannot model directly (DESIGN Β§10). The withhold + // stands in for that reader; it is not a claim that a single-invocation + // read can be stale. + sim.withholdNextEvent(1); + await slow.release(); + await fast.release(); + }, + // FAILS TODAY, identically to the unfenced scenario above β€” which is the + // finding. Turning the watermark on changes nothing here. + expect: { status: 'completed', output: 'afterSlow:doc-27' }, +}; diff --git a/workbench/sim-world/scenarios/step-vs-step-fork.ts b/workbench/sim-world/scenarios/step-vs-step-fork.ts new file mode 100644 index 0000000000..71e11394b8 --- /dev/null +++ b/workbench/sim-world/scenarios/step-vs-step-fork.ts @@ -0,0 +1,51 @@ +import type { ScenarioSpec } from '@workflow/world-sim'; + +export const scenario: ScenarioSpec = { + id: 'step-vs-step-fork', + name: 'corrupt: two racing STEPS, no hook anywhere', + description: + 'Answers "does this need an out-of-band event type?" β€” no. The fork is ' + + "decided by two of the run's own step_completed events, and withholding " + + 'one of them from the deciding read is enough. Two inline step bodies in ' + + 'ONE invocation are already two concurrent writers to the same log; no ' + + 'second invocation is required. ' + + 'Both writers are held so the ordering is stated rather than observed: ' + + 'stop `fast` and `slow` at their produced points, arm the withhold, then ' + + "release `slow` first, so the log's earliest completion is the one hidden " + + 'from the read that decides the fork. Hiding the *later* completion ' + + 'instead is harmless β€” the live pass then agrees with the log by ' + + 'accident β€” which is why the choice has to be made on purpose. ' + + 'Note that both waits are started before either is awaited: awaiting the ' + + 'first would let the second writer sail past its point.', + workflow: 'stepVsStepForkWorkflow', + input: ['doc-26'], + script: async (sim) => { + const fast = sim.writer.step('fast'); + const slow = sim.writer.step('slow'); + + const atFast = fast.runToEventProduced('step_completed'); + const atSlow = slow.runToEventProduced('step_completed'); + await atFast; + await atSlow; + + sim.check( + 'neither completion is in the log while both writers are held', + sim.world.events().filter((e) => e.eventType === 'step_completed') + .length === 0 + ); + + // Who this withheld reader is in production: not this invocation. With + // strongly-consistent reads a single invocation cannot miss its own + // committed write, so the reader that misses one of these two step + // writes is a *concurrent second invocation* of the same run β€” the storm + // shape, which the sim cannot model directly (DESIGN Β§10). The withhold + // stands in for that reader; it is not a claim that a single-invocation + // read can be stale. + sim.withholdNextEvent(1); + await slow.release(); + await fast.release(); + }, + // FAILS TODAY. `slow` commits first, so the log says `slow` won the race + // and the run should end on `afterSlow`. The live pass sees only `fast`. + expect: { status: 'completed', output: 'afterSlow:doc-26' }, +}; diff --git a/workbench/sim-world/scenarios/unclaimed-payload-under-fork.ts b/workbench/sim-world/scenarios/unclaimed-payload-under-fork.ts new file mode 100644 index 0000000000..430a242176 --- /dev/null +++ b/workbench/sim-world/scenarios/unclaimed-payload-under-fork.ts @@ -0,0 +1,107 @@ +import type { ScenarioSpec } from '@workflow/world-sim'; + +export const scenario: ScenarioSpec = { + id: 'unclaimed-payload-under-fork', + name: 'unclaimed hook payload sits between the fork and its wait', + description: + 'Three deliveries are pending in one resume: a hook payload nobody ' + + 'reads, a wait_completed the log orders next, and a step result last. ' + + 'A step result is allowed to skip the unclaimed payload β€” it would ' + + 'otherwise stall until the barrier registry idles β€” but skipping the ' + + 'wait parked behind that payload inverts the order the log recorded, ' + + 'and the two branches swap the step_created ids they draw next.', + workflow: 'unclaimedPayloadForkWorkflow', + input: ['doc-32'], + script: async (sim) => { + const wf = sim.writer.orchestrator(); + const body = sim.writer.step('pokedWork'); + + // 1. Hold the orchestrator inside the call that commits the wait, and land + // the payload there. Delivering it from outside that window is a race; + // delivering it from inside is a decision. + await wf.runToEventCommitted('wait_created'); + await sim.deliverHook('poke:doc-32', { kind: 'poke' }); + + // 2. Arm the hold on the step body *before* releasing the orchestrator. + // `runTo` is level-triggered and the body reaches its write during the + // release, so arming afterwards would be waiting for a point already + // gone by. + const atBody = body.runToEventProduced('step_completed'); + await wf.release(); + await atBody; + + // 3. The step result is now outstanding and the delivery loop is stopped + // inside the delivery waiting on it, so the watchdog can only fire from + // here. Hold that second delivery the instant its `wait_completed` is + // durable β€” pick the timer explicitly, because the hook delivery + // enqueued a flow message of its own and it sorts earlier. + const atWait = wf.runToEventCommitted('wait_completed'); + const fired = sim.deliverQueued( + (pending) => + pending.find((m) => m.readyAtMs > sim.world.nowMs())?.messageId + ); + await atWait; + + // 4. Land the step result immediately behind the wait, while the delivery + // that wrote the wait has not yet read the log back. That is what puts + // all three β€” unclaimed payload, armed wait, step result β€” in one + // barrier set, in that order. Firing the timer and releasing the step + // independently would give the same log with the wait's own branch + // already resumed, and nothing left to order. + const atCommitted = body.runToEventCommitted('step_completed'); + await body.release(); + await atCommitted; + await body.release(); + + await wf.release(); + sim.check('the watchdog fired while the step result was held', await fired); + + // 5. The property. Two branches were resolved by two events; the log puts + // one of those events first. Whichever branch that is must be the + // branch that resumes first, because resuming is what draws the next + // correlation id β€” and a replay has nothing but log order to go on. + // + // Note this is *not* the replay check. Replay runs the same code and so + // reproduces the same delivery order, agreeing with a log that is + // internally inconsistent. What breaks in production is a replay + // against a log some *other* build wrote, and the invariant that + // catches that here is the log disagreeing with itself. + const events = sim.world.events(); + // `eventData.stepName` is the fully qualified name and only `step_created` + // carries it, so go through the materialized step rows instead: a row's + // `stepId` is the correlation id every event of that step shares. + const correlationOf = (shortName: string) => + sim.world.steps().find((s) => s.stepName.endsWith(shortName))?.stepId; + const at = (eventType: string, shortName: string) => { + const correlationId = correlationOf(shortName); + return events.findIndex( + (e) => e.eventType === eventType && e.correlationId === correlationId + ); + }; + const waitResolved = events.findIndex( + (e) => e.eventType === 'wait_completed' + ); + const stepResolved = at('step_completed', 'pokedWork'); + const sleepBranchResumed = at('step_created', 'afterPokedSleep'); + const stepBranchResumed = at('step_created', 'afterPokedStep'); + + const waitWasResolvedFirst = waitResolved < stepResolved; + const sleepBranchResumedFirst = sleepBranchResumed < stepBranchResumed; + sim.check( + 'the branch resolved first by the log is the branch that resumes first', + waitWasResolvedFirst === sleepBranchResumedFirst + ); + }, + // Both branches run to completion in either delivery order, so the output is + // the same whichever one resumes first. That is the point: nothing about the + // result says which id each branch drew, and only the log-order check in + // step 5 can tell. (Not the replay β€” as step 5 explains, replay reruns the + // same code against the same log and agrees with it either way.) + // + // This scenario was red until #3406 fixed the delivery-barrier ordering; it + // is kept as the regression test for that fix. + expect: { + status: 'completed', + output: 'afterStep:doc-32|afterSleep:doc-32', + }, +}; diff --git a/workbench/sim-world/scenarios/writers-independent-step-bodies.ts b/workbench/sim-world/scenarios/writers-independent-step-bodies.ts new file mode 100644 index 0000000000..a9d006606f --- /dev/null +++ b/workbench/sim-world/scenarios/writers-independent-step-bodies.ts @@ -0,0 +1,45 @@ +import type { ScenarioSpec } from '@workflow/world-sim'; + +export const scenario: ScenarioSpec = { + id: 'writers-independent-step-bodies', + name: 'writers: two step bodies advance independently', + description: + 'The claim the whole writer API rests on: two inline step bodies in one ' + + "delivery are separately advanceable. Hold slow's step_completed at its " + + "produced (pre-commit) point; while it is held, fast's step_completed " + + 'still commits. Per-writer scheduling needs no new concurrency, only a ' + + 'way to name and steer what is already there. ' + + 'Note the assertion is level-triggered (read the log) rather than ' + + 'edge-triggered (await the event): fast commits during the hold itself, ' + + "because arming yields and fast's create was already in flight. An " + + '`until()` here waits for an edge that has already passed and deadlocks.', + workflow: 'stepVsStepForkWorkflow', + input: ['doc-28'], + script: async (sim) => { + const slow = sim.writer.step('slow'); + const held = await slow.runToEventProduced('step_completed'); + + const committed = sim.world + .events() + .filter((e) => e.eventType === 'step_completed'); + sim.check( + 'fast committed while slow was held pre-commit', + committed.some((e) => + String((e.eventData as { stepName?: string })?.stepName).endsWith( + '//fast' + ) + ) + ); + sim.check( + 'slow has NOT committed β€” it is the one being held', + !committed.some((e) => + String((e.eventData as { stepName?: string })?.stepName).endsWith( + '//slow' + ) + ) + ); + + await held.release(); + }, + expect: { status: 'completed' }, +}; diff --git a/workbench/sim-world/scenarios/writers-scripted-tempo.ts b/workbench/sim-world/scenarios/writers-scripted-tempo.ts new file mode 100644 index 0000000000..9e31f504f4 --- /dev/null +++ b/workbench/sim-world/scenarios/writers-scripted-tempo.ts @@ -0,0 +1,42 @@ +import type { ScenarioSpec } from '@workflow/world-sim'; + +export const scenario: ScenarioSpec = { + id: 'writers-scripted-tempo', + name: 'writers: the script names the tempo top to bottom', + description: + 'Every ordering in this scenario is a statement: hold the orchestrator ' + + 'at the call that commits step_started, assert what the log does and does ' + + 'not contain, deliver, release, then wait for the run to finish and ' + + 'assert the committed order. `until` is the read-only counterpart to a ' + + 'hold β€” it waits for a point without stopping anything.', + workflow: 'approvalWorkflow', + input: ['doc-7'], + script: async (sim) => { + const wf = sim.writer.orchestrator(); + // Nothing else in the world can advance this writer while it is held. + await wf.runToEventCommitted('step_started', 'reserveInventory'); + + sim.check( + 'the hook is registered but nothing has been received yet', + sim.world.events().some((e) => e.eventType === 'hook_created') && + !sim.world.events().some((e) => e.eventType === 'hook_received') + ); + + await sim.deliverHook('approval:doc-7', { + approved: true, + reviewer: 'hopper', + }); + await wf.release(); + + await sim.until({ eventType: 'run_completed', phase: 'after' }); + const order = sim.world.events().map((e) => e.eventType); + sim.check( + 'hook_received precedes step_completed', + order.indexOf('hook_received') < order.indexOf('step_completed') + ); + }, + expect: { + status: 'completed', + output: { status: 'settled:reserved:doc-7', reviewer: 'hopper' }, + }, +}; diff --git a/workbench/sim-world/tsconfig.json b/workbench/sim-world/tsconfig.json new file mode 100644 index 0000000000..c8a96b893a --- /dev/null +++ b/workbench/sim-world/tsconfig.json @@ -0,0 +1,9 @@ +{ + "extends": "@workflow/tsconfig/base.json", + "compilerOptions": { + "noEmit": true, + "allowImportingTsExtensions": true, + "types": ["node"] + }, + "include": ["run.ts", "scenarios", "workflows"] +} diff --git a/workbench/sim-world/workflows/index.ts b/workbench/sim-world/workflows/index.ts new file mode 100644 index 0000000000..3b876e06ac --- /dev/null +++ b/workbench/sim-world/workflows/index.ts @@ -0,0 +1,668 @@ +/** + * Every workflow the scenario book runs, in one file. + * + * They are here together because they are read together: a scenario names a + * workflow and a tempo, and checking whether the tempo is the interesting one + * means looking at the branch it steers. Splitting them across modules made + * that a two-file hop for no benefit β€” the bundle compiles the whole directory + * either way, and no workflow here imports another. + * + * Grouped by what the workflow is *for*, simplest first: + * + * 1. Smoke β€” the smallest logs a run can leave. + * 2. Timing β€” sleeps and retries, to prove virtual time costs nothing. + * 3. Approval β€” a hook and a step suspended together. + * 4. Peek β€” branching on whether a hook has *already* fired. + * 5. Attributes β€” the only mutable run state, written from two contexts. + * 6. Forks β€” a hook racing a timeout, which is where corruption lives. + * 7. Step-vs-step β€” the same fork with no out-of-band event at all. + * 8. Unclaimed β€” a hook payload nobody reads, parked under a fork. + * + * Step functions are module-private and sit directly above the first workflow + * that uses them. Names are unique across the whole file, so a scenario can + * steer a writer by short step name (`sim.writer.step('slow')`) without + * ambiguity. + */ + +import { + createHook, + getStepMetadata, + RetryableError, + setAttributes, + sleep, +} from 'workflow'; + +// --------------------------------------------------------------------------- +// 1. Smoke +// --------------------------------------------------------------------------- + +/** Nothing but a return value: the smallest log a completed run can have. */ +export async function emptyWorkflow() { + 'use workflow'; + return 'done'; +} + +async function noopStep() { + 'use step'; + return null; +} + +/** One step that does nothing. The smallest log that exercises the step path. */ +export async function oneStepWorkflow() { + 'use workflow'; + return await noopStep(); +} + +// --------------------------------------------------------------------------- +// 2. Timing +// --------------------------------------------------------------------------- + +async function prepare(input: string) { + 'use step'; + return `prepared:${input}`; +} + +async function finalize(input: string) { + 'use step'; + return `finalized:${input}`; +} + +/** + * A month-long sleep between two steps. + * + * Under virtual time this costs nothing: the wait continuation is a queue + * message dated 30 days out, and delivering it is a clock assignment. The + * scenario for this workflow is the proof that "all scenarios terminate" + * survives contact with realistic durations. + */ +export async function longSleepWorkflow(input: string) { + 'use workflow'; + + const prepared = await prepare(input); + await sleep('30d'); + return await finalize(prepared); +} + +/** + * Fails deterministically for its first two attempts, then succeeds. + * + * Retry backoff is `delaySeconds` on a queue message, so the retry schedule + * is virtual too β€” the scenario observes three `step_started` events and the + * growing gaps between them without waiting for any of them. + */ +async function flakyStep(label: string) { + 'use step'; + const { attempt } = getStepMetadata(); + if (attempt < 3) { + throw new RetryableError(`${label} failed on attempt ${attempt}`); + } + return `${label}:ok-on-attempt-${attempt}`; +} + +export async function retryingWorkflow(label: string) { + 'use workflow'; + return await flakyStep(label); +} + +/** + * Two steps that suspend together, so the world sees interleaved + * `step_started` / `step_completed` pairs for distinct correlation IDs. + */ +export async function parallelStepsWorkflow(input: string) { + 'use workflow'; + const [a, b] = await Promise.all([prepare(input), finalize(input)]); + return `${a}|${b}`; +} + +// --------------------------------------------------------------------------- +// 3. Approval β€” a hook and a step suspended together +// --------------------------------------------------------------------------- + +async function reserveInventory(documentId: string) { + 'use step'; + return `reserved:${documentId}`; +} + +async function settleOrder(reservation: string, approved: boolean) { + 'use step'; + return approved ? `settled:${reservation}` : `released:${reservation}`; +} + +/** + * A step and a hook suspend together. + * + * This is the shape the timing control exists for: the run has an in-flight + * step *and* an open hook at the same moment, so where the `hook_received` + * event lands relative to `step_started` / `step_completed` is a real + * ordering choice rather than an artifact of whoever won the race. A + * scenario pins that choice with a cue on the exact world call that commits + * the step event. + */ +export async function approvalWorkflow(documentId: string) { + 'use workflow'; + + using hook = createHook<{ approved: boolean; reviewer: string }>({ + token: `approval:${documentId}`, + }); + + const [reservation, decision] = await Promise.all([ + reserveInventory(documentId), + hook, + ]); + + const status = await settleOrder(reservation, decision.approved); + return { status, reviewer: decision.reviewer }; +} + +/** + * Approval with a deadline: whichever of the hook and the timer resolves + * first decides the outcome. + * + * Under a real world this is genuinely racy and therefore untestable; here + * the hook only arrives if a cue delivers it, and the timer only fires when + * the scheduler jumps the clock to it, so both branches are reachable on + * demand. + */ +export async function approvalWithDeadlineWorkflow( + documentId: string, + deadline: string +) { + 'use workflow'; + + using hook = createHook<{ approved: boolean }>({ + token: `approval:${documentId}`, + }); + + const decision = await Promise.race([ + hook.then((payload) => (payload.approved ? 'approved' : 'rejected')), + sleep(deadline as never).then(() => 'timed-out' as const), + ]); + + return decision; +} + +/** + * Waits on a hook with nothing else to wake it. Used to pin down what the + * simulator does when external input never arrives: report a stall with the + * open hook named, rather than hang. + */ +export async function blockedOnHookWorkflow(documentId: string) { + 'use workflow'; + + using hook = createHook<{ approved: boolean }>({ + token: `approval:${documentId}`, + }); + + const decision = await hook; + return decision.approved; +} + +/** + * Two sequential steps, then the hook is awaited. + * + * Useful for cues keyed on *execution state* rather than on a single event: + * "deliver once both steps have completed" is a predicate over the world, and + * it lands the payload before the workflow ever awaits the hook. + */ +export async function stagedApprovalWorkflow(documentId: string) { + 'use workflow'; + + using hook = createHook<{ approved: boolean }>({ + token: `approval:${documentId}`, + }); + + const reservation = await reserveInventory(documentId); + const settled = await settleOrder(reservation, true); + const decision = await hook; + + return `${settled}/${decision.approved ? 'confirmed' : 'reverted'}`; +} + +// --------------------------------------------------------------------------- +// 4. Peek β€” branching on whether a hook has *already* fired +// --------------------------------------------------------------------------- + +async function reserve(documentId: string) { + 'use step'; + return `reserved:${documentId}`; +} + +async function shipWithoutApproval(reservation: string) { + 'use step'; + return `shipped-unapproved:${reservation}`; +} + +async function shipWithApproval(reservation: string) { + 'use step'; + return `shipped-approved:${reservation}`; +} + +/** + * Branches on whether a hook has *already* fired, without waiting for it. + * + * There is no peek API on `Hook`, so the way a user writes "has the approval + * landed yet?" is to race it against an already-resolved promise. That makes + * the branch a function of *when* the payload arrived rather than of the + * payload itself β€” and "when" is the one thing a replay does not reproduce, + * because on replay the whole log is already there. + * + * The hazard: if `hook_received` is committed at a log position before the + * branch's own events, then a replay reaching this race has the payload + * buffered and takes the other fork. The first execution shipped without + * approval; the replay wants to ship with it, and the log says otherwise. + */ +export async function hookPeekWorkflow(documentId: string) { + 'use workflow'; + + using hook = createHook<{ approved: boolean }>({ + token: `peek:${documentId}`, + }); + + const reservation = await reserve(documentId); + + const peeked = await Promise.race([ + hook.then(() => 'arrived' as const), + Promise.resolve('not-yet' as const), + ]); + + return peeked === 'arrived' + ? await shipWithApproval(reservation) + : await shipWithoutApproval(reservation); +} + +async function probe(documentId: string) { + 'use step'; + return `probed:${documentId}`; +} + +/** + * The same branch, but racing the hook against a *step* rather than against an + * already-resolved promise. + * + * This is the harder version. A resolved promise wins on a microtask and the + * hook payload is deliberately deferred behind a macrotask, so the peek above + * always reads "not yet". Here both competitors are event-log deliveries, so + * which one wins is decided by the runtime's delivery-barrier ordering β€” and + * that ordering is keyed on log position, which the cue controls. + */ +export async function hookRaceStepWorkflow(documentId: string) { + 'use workflow'; + + using hook = createHook<{ approved: boolean }>({ + token: `race:${documentId}`, + }); + + const winner = await Promise.race([ + hook.then(() => 'hook' as const), + probe(documentId).then(() => 'step' as const), + ]); + + return winner === 'hook' + ? await shipWithApproval(`race:${documentId}`) + : await shipWithoutApproval(`race:${documentId}`); +} + +// --------------------------------------------------------------------------- +// 5. Attributes β€” the only mutable run state, written from two contexts +// --------------------------------------------------------------------------- + +/** + * Two concurrent branches: one gated on a hook that then records the decision + * as run state, one an ordinary step. + * + * Three delivery types land in one log here β€” a step result, a hook payload, + * and an `attr_set` β€” and the hook's arrival time decides the order of the last + * two relative to the first. Attributes are the only *mutable* run state a + * workflow can write, and the world materializes them by folding the log, so + * this is where an ordering bug would show up as a wrong final value rather + * than as a divergence. + */ +export async function concurrentAttributeWorkflow(documentId: string) { + 'use workflow'; + + using hook = createHook<{ approved: boolean }>({ + token: `attr:${documentId}`, + }); + + const [approved, probed] = await Promise.all([ + (async () => { + const payload = await hook; + await setAttributes({ approval: payload.approved ? 'yes' : 'no' }); + return payload.approved; + })(), + probe(documentId), + ]); + + await setAttributes({ phase: 'settled' }); + return `${probed}/${approved ? 'approved' : 'rejected'}`; +} + +/** Writes run state from inside a step body, not from the orchestrator. */ +async function probeAndRecord(documentId: string) { + 'use step'; + await setAttributes({ probedBy: 'step', document: documentId }); + return `recorded:${documentId}`; +} + +/** + * Two concurrent steps plus a hook, where the attribute is written from *step* + * context rather than from the orchestrator. + * + * A different path than `concurrentAttributeWorkflow`: an `attr_set` from a step + * carries `writer: { type: 'step', stepId, attempt }`, is committed inline while + * the body runs rather than batched at the next suspension, and gets no + * correlationId dedupe β€” so its log position really is decided by step timing. + */ +export async function stepAttributeWorkflow(documentId: string) { + 'use workflow'; + + using hook = createHook<{ approved: boolean }>({ + token: `stepattr:${documentId}`, + }); + + const [payload, recorded, probed] = await Promise.all([ + hook, + probeAndRecord(documentId), + probe(documentId), + ]); + + return `${recorded}|${probed}|${payload.approved ? 'yes' : 'no'}`; +} + +// --------------------------------------------------------------------------- +// 6. Forks β€” a hook racing a timeout +// --------------------------------------------------------------------------- + +/** Step 1. Does nothing; it exists to put a step boundary before the race. */ +async function stepOne() { + 'use step'; + return null; +} + +/** Step 2 β€” the "hook arrived" branch. */ +async function stepTwo(documentId: string) { + 'use step'; + return `step2:${documentId}`; +} + +/** Step 3 β€” the "timed out, no hook" branch. */ +async function stepThree(documentId: string) { + 'use step'; + return `step3:${documentId}`; +} + +/** + * step 1 β†’ wait for the hook with a timeout β†’ branch on which won. + * + * The dangerous window is between `wait_completed` (the timeout firing) and the + * commit of whichever branch step gets chosen. A payload delivered there is + * durably *ahead* of the branch in the log while the first execution decided + * the branch without it β€” so a replay reaching the race sees both competitors + * resolvable and has to pick the same one, on log position alone. + */ +export async function hookTimeoutForkWorkflow(documentId: string) { + 'use workflow'; + + using hook = createHook<{ approved: boolean }>({ + token: `fork:${documentId}`, + }); + + await stepOne(); + + const arrived = await Promise.race([ + hook.then(() => true), + sleep('1m').then(() => false), + ]); + + return arrived ? await stepTwo(documentId) : await stepThree(documentId); +} + +async function settle(documentId: string) { + 'use step'; + return `settled:${documentId}`; +} + +async function recoverFirst(documentId: string) { + 'use step'; + return `recovered:${documentId}`; +} + +async function recoverSecond(previous: string) { + 'use step'; + return `${previous}+second`; +} + +async function reconcile(tail: string) { + 'use step'; + return `reconciled(${tail})`; +} + +/** + * The same fork, but the two paths emit a *different number of steps*. + * + * This is the amplifier the shape above was missing. Correlation IDs are + * positional ordinals of one seeded sequence, so when the settle path emits one + * step and the recovery path emits two, a replay that flips the branch renames + * every entity after the fork. The log then contains a `step_created` nobody + * asks for, which is an unrecoverable divergence rather than a benign retry + * that happens to mint the same ids. + * + * `reconcile` exists to carry the shift past the fork: its ordinal differs by + * one between the two paths. + */ +export async function stepCountForkWorkflow(documentId: string) { + 'use workflow'; + + using hook = createHook<{ approved: boolean }>({ + token: `count:${documentId}`, + }); + + await stepOne(); + + const arrived = await Promise.race([ + hook.then(() => true), + sleep('1m').then(() => false), + ]); + + let tail: string; + if (arrived) { + tail = await recoverSecond(await recoverFirst(documentId)); + } else { + tail = await settle(documentId); + } + + return await reconcile(tail); +} + +/** + * The same fork again, with one addition: the run suspends *after* the branch. + * + * Every check a write can meet happens inside the write itself β€” the fence is + * a conditional append, evaluated against the log as it stands at that + * instant. So the only moment at which a late-committing event can land + * without meeting any check at all is one where the run is making no writes: + * the gap between one delivery ending and the next beginning. This workflow + * creates such a gap in the middle of a run, which is where the third + * in-flight scenario lands its hook. + * + * Suspending after the branch (rather than letting the run finish) also keeps + * the hook alive. A run that has completed has disposed its hooks and gone + * terminal, and a `hook_received` arriving then is refused for reasons that + * have nothing to do with concurrency β€” which would hide the hazard rather + * than test it. + */ +export async function lateAppendForkWorkflow(documentId: string) { + 'use workflow'; + + using hook = createHook<{ approved: boolean }>({ + token: `count:${documentId}`, + }); + + await stepOne(); + + const arrived = await Promise.race([ + hook.then(() => true), + sleep('1m').then(() => false), + ]); + + const tail = arrived + ? await recoverSecond(await recoverFirst(documentId)) + : await settle(documentId); + + // The quiescent window. `wait_created` is the last write this delivery + // makes; nothing of this run is checked again until the timer fires. + await sleep('1m'); + + return await reconcile(tail); +} + +// --------------------------------------------------------------------------- +// 7. Step-vs-step β€” the same fork with no out-of-band event at all +// --------------------------------------------------------------------------- + +/** + * A fork decided entirely by events the run writes itself β€” no hook, no + * external writer, no timer. + * + * Two steps suspend together and race. The winner is whichever `step_completed` + * sits earlier in the log, so the branch is a function of log order exactly as + * the hook races were. The point of this shape is to show that "the log and the + * execution disagree" does not require an out-of-band event type: it requires + * only two events whose relative order decides a branch, plus a reader that + * missed one of them. + */ + +async function fast(documentId: string) { + 'use step'; + return `fast:${documentId}`; +} + +async function slow(documentId: string) { + 'use step'; + return `slow:${documentId}`; +} + +async function afterFast(documentId: string) { + 'use step'; + return `afterFast:${documentId}`; +} + +async function afterSlow(documentId: string) { + 'use step'; + return `afterSlow:${documentId}`; +} + +export async function stepVsStepForkWorkflow(documentId: string) { + 'use workflow'; + + const winner = await Promise.race([ + fast(documentId).then(() => 'fast' as const), + slow(documentId).then(() => 'slow' as const), + ]); + + return winner === 'fast' + ? await afterFast(documentId) + : await afterSlow(documentId); +} + +// --------------------------------------------------------------------------- +// 8. Unclaimed payload β€” a hook nobody reads, sitting under the fork +// --------------------------------------------------------------------------- + +/** + * A hook payload registers its delivery barrier **unarmed** when no branch is + * waiting on it (`workflow/hook.ts`, `armed: promises.length > 0`), because + * nothing in the workflow will ever resolve it β€” only the barrier registry's + * idle safety net retires it. Every other delivery that defers behind hooks + * therefore parks behind that payload, waits included. + * + * A step result may skip an unclaimed payload, or it would stall until that + * net fires. The hazard is that the skip can be *transitive*: the step also + * skips a `wait_completed` that is merely parked behind the payload, even + * though the wait sits earlier in the log and would otherwise gate it. The two + * branches then draw each other's correlation ids, and the log stops replaying + * into the run that wrote it. + * + * The shape below is the smallest thing that has all three barriers pending in + * one delivery: an unclaimed payload, an armed wait, and a step result, in + * that log order. It is written as `Promise.all` rather than `Promise.race` + * because the fault is not which branch *wins* β€” both branches run either way, + * and the output is the same β€” it is which branch resumes first and therefore + * which `step_created` id each one draws. That is invisible in the output and + * visible only to a replay. + */ + +async function pokedWork(documentId: string) { + 'use step'; + return `worked:${documentId}`; +} + +async function afterPokedStep(documentId: string) { + 'use step'; + return `afterStep:${documentId}`; +} + +async function afterPokedSleep(documentId: string) { + 'use step'; + return `afterSleep:${documentId}`; +} + +export async function unclaimedPayloadForkWorkflow(documentId: string) { + 'use workflow'; + + // Created and never read. Everything about this scenario follows from that. + using _poke = createHook<{ kind: string }>({ token: `poke:${documentId}` }); + + const branchStep = (async () => { + await pokedWork(documentId); + return await afterPokedStep(documentId); + })(); + + const branchSleep = (async () => { + await sleep('1m'); + return await afterPokedSleep(documentId); + })(); + + const [stepTail, sleepTail] = await Promise.all([branchStep, branchSleep]); + return `${stepTail}|${sleepTail}`; +} + +/** + * The control: the same three events in the same log order, with one branch + * awaiting the payload. + * + * Claiming it arms the hook barrier, so the wait no longer parks behind an + * entry that cannot resolve itself, and the step result gates on the wait the + * ordinary way. Everything else β€” the steps, the sleep, the tempo the scenario + * scripts β€” is identical, which is what makes the pair a controlled + * comparison rather than two unrelated runs. + */ +export async function claimedPayloadForkWorkflow(documentId: string) { + 'use workflow'; + + using poke = createHook<{ kind: string }>({ token: `poke:${documentId}` }); + + const branchStep = (async () => { + await pokedWork(documentId); + return await afterPokedStep(documentId); + })(); + + const branchSleep = (async () => { + await sleep('1m'); + return await afterPokedSleep(documentId); + })(); + + // Draws no correlation id of its own, so both workflows leave the same log + // shape; the only difference is that this one has a consumer attached when + // the payload lands. + const branchPoke = (async () => { + await poke; + })(); + + const [stepTail, sleepTail] = await Promise.all([ + branchStep, + branchSleep, + branchPoke, + ]); + return `${stepTail}|${sleepTail}`; +}