From 3811f85c3c9ae6245e3db00472404ef10a0336c1 Mon Sep 17 00:00:00 2001 From: Peter Wielander Date: Thu, 30 Jul 2026 14:15:07 -0700 Subject: [PATCH 01/35] feat(core): per-kind correlation ids and run-scoped queue idempotency keys MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Adds the client half of slot-based event identity, dormant until a run is stamped at spec version 6. - `@workflow/world` gains `slot-identity.ts` (26-char zero-padded decimal bodies, so a slot is a valid ULID body everywhere a ULID is accepted), `SPEC_VERSION_SLOT_IDENTITY` and `usesSlotIdentity()`. - `nextCorrelationId(kind)` on the orchestrator context is now the only place step and wait correlation ids are minted. In slot mode each kind counts independently from 1; otherwise it delegates to the same seeded ULID generator as before, so existing runs are byte-identical. - Every queue idempotency key is scoped to the run. Queues dedupe per queue, which is per workflow, so slot-numbered correlation ids would otherwise collide across concurrent runs of one workflow — silently, because a deduped send is answered normally and never dispatched. A source-scan test asserts no site builds a key any other way. - `packages/world` had test files but no `test` script, so 10 files never ran in CI. Added one. --- .changeset/slot-correlation-ids.md | 6 ++ packages/core/src/abort-consistency.test.ts | 5 + packages/core/src/abort-controller.test.ts | 5 + .../core/src/abort-replay-ordering.test.ts | 5 + .../async-deserialization-ordering.test.ts | 5 + packages/core/src/correlation-ids.test.ts | 92 +++++++++++++++++++ packages/core/src/correlation-ids.ts | 65 +++++++++++++ .../src/delivery-barrier-coverage.test.ts | 5 + .../core/src/hook-sleep-interaction.test.ts | 5 + packages/core/src/private.ts | 8 ++ packages/core/src/runtime.ts | 19 +++- .../core/src/runtime/idempotency-key.test.ts | 85 +++++++++++++++++ packages/core/src/runtime/idempotency-key.ts | 33 +++++++ .../core/src/runtime/step-ownership.test.ts | 43 ++++++--- packages/core/src/runtime/step-ownership.ts | 22 ++++- .../src/runtime/wait-continuation.test.ts | 69 ++++++++++---- .../core/src/runtime/wait-continuation.ts | 20 +++- .../core/src/step-delivery-hop-count.test.ts | 5 + .../core/src/step-delivery-ordering.test.ts | 5 + .../src/step-hydration-memoization.test.ts | 5 + packages/core/src/step.test.ts | 5 + packages/core/src/step.ts | 2 +- packages/core/src/workflow.ts | 10 +- packages/core/src/workflow/hook.test.ts | 5 + packages/core/src/workflow/sleep.test.ts | 5 + packages/core/src/workflow/sleep.ts | 2 +- packages/world/package.json | 1 + packages/world/src/index.ts | 9 ++ packages/world/src/slot-identity.test.ts | 59 ++++++++++++ packages/world/src/slot-identity.ts | 63 +++++++++++++ packages/world/src/spec-version.ts | 30 ++++++ 31 files changed, 651 insertions(+), 47 deletions(-) create mode 100644 .changeset/slot-correlation-ids.md create mode 100644 packages/core/src/correlation-ids.test.ts create mode 100644 packages/core/src/correlation-ids.ts create mode 100644 packages/core/src/runtime/idempotency-key.test.ts create mode 100644 packages/core/src/runtime/idempotency-key.ts create mode 100644 packages/world/src/slot-identity.test.ts create mode 100644 packages/world/src/slot-identity.ts diff --git a/.changeset/slot-correlation-ids.md b/.changeset/slot-correlation-ids.md new file mode 100644 index 0000000000..e99c6c6578 --- /dev/null +++ b/.changeset/slot-correlation-ids.md @@ -0,0 +1,6 @@ +--- +'@workflow/core': patch +'@workflow/world': patch +--- + +Scope every queue idempotency key to the run, and number step and wait correlation IDs per kind so that inserting one kind no longer renumbers the others. diff --git a/packages/core/src/abort-consistency.test.ts b/packages/core/src/abort-consistency.test.ts index fcd5ed09f7..5eb1c35905 100644 --- a/packages/core/src/abort-consistency.test.ts +++ b/packages/core/src/abort-consistency.test.ts @@ -11,6 +11,7 @@ import type { Event, WorkflowRun } from '@workflow/world'; import * as nanoid from 'nanoid'; import { monotonicFactory } from 'ulid'; import { describe, expect, it, vi } from 'vitest'; +import { createCorrelationIdFactory } from './correlation-ids.js'; import { EventsConsumer } from './events-consumer.js'; import type { WorkflowSuspension } from './global.js'; import type { WorkflowOrchestratorContext } from './private.js'; @@ -45,6 +46,10 @@ function setupWorkflowContext(events: Event[]): WorkflowOrchestratorContext { }), invocationsQueue: new Map(), generateUlid: () => ulid(workflowStartedAt), + nextCorrelationId: createCorrelationIdFactory({ + specVersion: undefined, + generateUlid: () => ulid(workflowStartedAt), + }), generateNanoid: nanoid.customRandom(nanoid.urlAlphabet, 21, (size) => new Uint8Array(size).map(() => 256 * context.globalThis.Math.random()) ), diff --git a/packages/core/src/abort-controller.test.ts b/packages/core/src/abort-controller.test.ts index 580bfe3719..4d1cbef1a5 100644 --- a/packages/core/src/abort-controller.test.ts +++ b/packages/core/src/abort-controller.test.ts @@ -12,6 +12,7 @@ import type { Event } from '@workflow/world'; import * as nanoid from 'nanoid'; import { monotonicFactory } from 'ulid'; import { describe, expect, it, vi } from 'vitest'; +import { createCorrelationIdFactory } from './correlation-ids.js'; import { DEFERRED_CHECK_DELAY_MS, EventsConsumer } from './events-consumer.js'; import type { WorkflowOrchestratorContext } from './private.js'; import { ReplayPayloadCache } from './replay-payload-cache.js'; @@ -43,6 +44,10 @@ function setupWorkflowContext( }), invocationsQueue: new Map(), generateUlid: () => ulid(workflowStartedAt), + nextCorrelationId: createCorrelationIdFactory({ + specVersion: undefined, + generateUlid: () => ulid(workflowStartedAt), + }), generateNanoid: nanoid.customRandom(nanoid.urlAlphabet, 21, (size) => new Uint8Array(size).map(() => 256 * context.globalThis.Math.random()) ), diff --git a/packages/core/src/abort-replay-ordering.test.ts b/packages/core/src/abort-replay-ordering.test.ts index 8ddf4aec03..2708380899 100644 --- a/packages/core/src/abort-replay-ordering.test.ts +++ b/packages/core/src/abort-replay-ordering.test.ts @@ -27,6 +27,7 @@ import type { Event } from '@workflow/world'; import * as nanoid from 'nanoid'; import { monotonicFactory } from 'ulid'; import { describe, expect, it, vi } from 'vitest'; +import { createCorrelationIdFactory } from './correlation-ids.js'; import { EventsConsumer } from './events-consumer.js'; import { scheduleWhenIdle, @@ -78,6 +79,10 @@ function setupWorkflowContext(events: Event[]): WorkflowOrchestratorContext { }), invocationsQueue: new Map(), generateUlid: () => ulid(workflowStartedAt), + nextCorrelationId: createCorrelationIdFactory({ + specVersion: undefined, + generateUlid: () => ulid(workflowStartedAt), + }), generateNanoid: nanoid.customRandom(nanoid.urlAlphabet, 21, (size) => new Uint8Array(size).map(() => 256 * context.globalThis.Math.random()) ), diff --git a/packages/core/src/async-deserialization-ordering.test.ts b/packages/core/src/async-deserialization-ordering.test.ts index 7a1ff1d346..4dc20fd9a6 100644 --- a/packages/core/src/async-deserialization-ordering.test.ts +++ b/packages/core/src/async-deserialization-ordering.test.ts @@ -4,6 +4,7 @@ import * as nanoid from 'nanoid'; import { monotonicFactory } from 'ulid'; import { afterEach, beforeAll, describe, expect, it, vi } from 'vitest'; import { registerSerializationClass } from './class-serialization.js'; +import { createCorrelationIdFactory } from './correlation-ids.js'; import { EventsConsumer } from './events-consumer.js'; import type { WorkflowOrchestratorContext } from './private.js'; import { ReplayPayloadCache } from './replay-payload-cache.js'; @@ -58,6 +59,10 @@ function setupWorkflowContext(events: Event[]): WorkflowOrchestratorContext { }), invocationsQueue: new Map(), generateUlid: () => ulid(workflowStartedAt), + nextCorrelationId: createCorrelationIdFactory({ + specVersion: undefined, + generateUlid: () => ulid(workflowStartedAt), + }), generateNanoid: nanoid.customRandom(nanoid.urlAlphabet, 21, (size) => new Uint8Array(size).map(() => 256 * context.globalThis.Math.random()) ), diff --git a/packages/core/src/correlation-ids.test.ts b/packages/core/src/correlation-ids.test.ts new file mode 100644 index 0000000000..f4771e6d7a --- /dev/null +++ b/packages/core/src/correlation-ids.test.ts @@ -0,0 +1,92 @@ +import { + FIRST_SLOT, + SLOT_ID_WIDTH, + SPEC_VERSION_CURRENT, + SPEC_VERSION_SLOT_IDENTITY, + slotFromId, +} from '@workflow/world'; +import { describe, expect, it } from 'vitest'; +import { createCorrelationIdFactory } from './correlation-ids.js'; + +/** Stands in for the invocation's seeded, replay-stable ULID generator. */ +function fakeUlids(): () => string { + let issued = 0; + return () => `01ULID${String(++issued).padStart(20, '0')}`; +} + +function slotFactory() { + return createCorrelationIdFactory({ + specVersion: SPEC_VERSION_SLOT_IDENTITY, + generateUlid: fakeUlids(), + }); +} + +describe('createCorrelationIdFactory', () => { + describe('slot identity', () => { + it('numbers each kind densely from the first slot', () => { + const next = slotFactory(); + expect(slotFromId(next('step'))).toBe(FIRST_SLOT); + expect(slotFromId(next('step'))).toBe(FIRST_SLOT + 1); + expect(slotFromId(next('wait'))).toBe(FIRST_SLOT); + }); + + it('keeps the prefix of the kind it was asked for', () => { + const next = slotFactory(); + expect(next('step')).toMatch( + new RegExp(`^step_[0-9]{${SLOT_ID_WIDTH}}$`) + ); + expect(next('wait')).toMatch( + new RegExp(`^wait_[0-9]{${SLOT_ID_WIDTH}}$`) + ); + }); + + it('issues the same sequence to two fresh invocations', () => { + // Replay stability: the VM is rebuilt per replay and the workflow body + // issues its operations in the same order, so nothing needs seeding. + const replay = () => { + const next = slotFactory(); + return [next('step'), next('wait'), next('step'), next('step')]; + }; + expect(replay()).toEqual(replay()); + }); + + it('does not renumber steps or waits when another kind allocates', () => { + // Kinds that stay on ULIDs (hooks, attributes) draw from generateUlid, + // and a per-kind counter means interleaving them cannot shift a step's + // number — which a single shared sequence would. + const ulids = fakeUlids(); + const next = createCorrelationIdFactory({ + specVersion: SPEC_VERSION_SLOT_IDENTITY, + generateUlid: ulids, + }); + const firstStep = next('step'); + ulids(); + ulids(); + const secondStep = next('step'); + expect(slotFromId(firstStep)).toBe(FIRST_SLOT); + expect(slotFromId(secondStep)).toBe(FIRST_SLOT + 1); + expect(slotFromId(next('wait'))).toBe(FIRST_SLOT); + }); + }); + + describe('ULID identity', () => { + it('draws from the invocation generator for every kind', () => { + const next = createCorrelationIdFactory({ + specVersion: SPEC_VERSION_CURRENT, + generateUlid: fakeUlids(), + }); + // One shared sequence, exactly as before slots existed: an id's number + // reflects the order of allocation across all kinds. + expect(next('step')).toBe('step_01ULID00000000000000000001'); + expect(next('wait')).toBe('wait_01ULID00000000000000000002'); + }); + + it('treats a run with no spec version as ULID-numbered', () => { + const next = createCorrelationIdFactory({ + specVersion: undefined, + generateUlid: fakeUlids(), + }); + expect(slotFromId(next('step'))).toBeUndefined(); + }); + }); +}); diff --git a/packages/core/src/correlation-ids.ts b/packages/core/src/correlation-ids.ts new file mode 100644 index 0000000000..dc5d6a69fc --- /dev/null +++ b/packages/core/src/correlation-ids.ts @@ -0,0 +1,65 @@ +/** + * Correlation-id allocation for a single workflow invocation. + * + * A correlation id names an operation the workflow body issued — a step call, a + * sleep — and must come out identical on every replay of that run, because it + * is how a replay recognises the event that already recorded the operation. + * + * Two schemes exist. A run on ULID identity draws from the invocation's seeded + * monotonic ULID generator, which is replay-stable because its seed and initial + * clock are derived from the run. A run on slot identity counts: the first step + * of the run is `step_…001`, the second `step_…002`, zero-padded to ULID width + * (see `slotIdBody`). Which scheme applies is fixed by the run's persisted + * `specVersion` and never by the build, so a run started before slot identity + * keeps proposing the ids its log already holds. + * + * Counters are **per kind**, and there is nothing to seed them with. The VM is + * rebuilt for every replay and the workflow body issues its operations in the + * same order every time, which is the same argument that licenses the seeded + * ULID sequence today. Recovering counters from the loaded log would be actively + * wrong: the n-th step's id would then depend on how much of the log this + * replay happened to load. + * + * Per-kind is a strict improvement over the shared ULID sequence. Today all + * four id kinds draw from one generator, so introducing a hook allocation + * renumbers every step and wait issued after it; separate counters mean a step's + * number depends only on the steps before it. + * + * Hook and attribute ids stay on ULIDs and are not allocated here. Both are + * written from outside the VM in cases where no counter exists (an attribute set + * on a run from the outside), and both already carry their own per-run + * idempotency, so slots would buy them nothing. + */ + +import { FIRST_SLOT, slotIdBody, usesSlotIdentity } from '@workflow/world'; + +/** Operation kinds whose correlation ids are allocated per run and per kind. */ +export type CorrelationKind = 'step' | 'wait'; + +/** + * Allocates the next correlation id for a kind, prefix included. Returning the + * finished id — rather than a number or a bare body — keeps the prefix from + * ever diverging from the counter it was drawn against. + */ +export type CorrelationIdFactory = (kind: CorrelationKind) => string; + +export function createCorrelationIdFactory({ + specVersion, + generateUlid, +}: { + /** The run's *persisted* spec version. */ + specVersion: number | undefined; + /** The invocation's replay-stable ULID generator. */ + generateUlid: () => string; +}): CorrelationIdFactory { + if (!usesSlotIdentity(specVersion)) { + return (kind) => `${kind}_${generateUlid()}`; + } + + const allocated = new Map(); + return (kind) => { + const slot = (allocated.get(kind) ?? FIRST_SLOT - 1) + 1; + allocated.set(kind, slot); + return `${kind}_${slotIdBody(slot)}`; + }; +} diff --git a/packages/core/src/delivery-barrier-coverage.test.ts b/packages/core/src/delivery-barrier-coverage.test.ts index 836d7d3948..6b97f00092 100644 --- a/packages/core/src/delivery-barrier-coverage.test.ts +++ b/packages/core/src/delivery-barrier-coverage.test.ts @@ -45,6 +45,7 @@ import type { Event } from '@workflow/world'; import * as nanoid from 'nanoid'; import { monotonicFactory } from 'ulid'; import { describe, expect, it, vi } from 'vitest'; +import { createCorrelationIdFactory } from './correlation-ids.js'; import { EventsConsumer } from './events-consumer.js'; import { WorkflowSuspension } from './global.js'; import { @@ -86,6 +87,10 @@ function setupWorkflowContext(events: Event[]): WorkflowOrchestratorContext { }), invocationsQueue: new Map(), generateUlid: () => ulid(workflowStartedAt), + nextCorrelationId: createCorrelationIdFactory({ + specVersion: undefined, + generateUlid: () => ulid(workflowStartedAt), + }), generateNanoid: nanoid.customRandom(nanoid.urlAlphabet, 21, (size) => new Uint8Array(size).map(() => 256 * context.globalThis.Math.random()) ), diff --git a/packages/core/src/hook-sleep-interaction.test.ts b/packages/core/src/hook-sleep-interaction.test.ts index 29d64fcc58..c02d36ad19 100644 --- a/packages/core/src/hook-sleep-interaction.test.ts +++ b/packages/core/src/hook-sleep-interaction.test.ts @@ -4,6 +4,7 @@ import type { Event } from '@workflow/world'; import * as nanoid from 'nanoid'; import { monotonicFactory } from 'ulid'; import { afterEach, describe, expect, it, vi } from 'vitest'; +import { createCorrelationIdFactory } from './correlation-ids.js'; import { EventsConsumer } from './events-consumer.js'; import { WorkflowSuspension } from './global.js'; import type { WorkflowOrchestratorContext } from './private.js'; @@ -58,6 +59,10 @@ function setupWorkflowContext(events: Event[]): WorkflowOrchestratorContext { }), invocationsQueue: new Map(), generateUlid: () => ulid(workflowStartedAt), + nextCorrelationId: createCorrelationIdFactory({ + specVersion: undefined, + generateUlid: () => ulid(workflowStartedAt), + }), generateNanoid: nanoid.customRandom(nanoid.urlAlphabet, 21, (size) => new Uint8Array(size).map(() => 256 * context.globalThis.Math.random()) ), diff --git a/packages/core/src/private.ts b/packages/core/src/private.ts index 16fb6a3e4e..a4498b3179 100644 --- a/packages/core/src/private.ts +++ b/packages/core/src/private.ts @@ -4,6 +4,7 @@ import { withResolvers } from '@workflow/utils'; import type { WorldCapabilities } from '@workflow/world'; +import type { CorrelationIdFactory } from './correlation-ids.js'; import type { EventsConsumer } from './events-consumer.js'; import type { QueueItem } from './global.js'; import type { ReplayPayloadCache } from './replay-payload-cache.js'; @@ -142,6 +143,13 @@ export interface WorkflowOrchestratorContext { */ invocationsQueue: Map; onWorkflowError: (error: Error) => void; + /** + * Allocates the correlation id for a step or wait the workflow body just + * issued. Replay-stable, and the only place those ids are minted — see + * `correlation-ids.ts` for why the two kinds count separately and why + * nothing seeds them from the loaded log. + */ + nextCorrelationId: CorrelationIdFactory; generateUlid: () => string; generateNanoid: () => string; /** diff --git a/packages/core/src/runtime.ts b/packages/core/src/runtime.ts index 20094392fa..b8474d2206 100644 --- a/packages/core/src/runtime.ts +++ b/packages/core/src/runtime.ts @@ -61,6 +61,7 @@ import { withHealthCheck, withPreconditionRetry, } from './runtime/helpers.js'; +import { runScopedKey } from './runtime/idempotency-key.js'; import { handleReplayBudgetExhausted, ReplayBudget, @@ -1934,7 +1935,10 @@ export function workflowEntrypoint( }, { delaySeconds: backstopDelaySeconds, - idempotencyKey: backstopIdempotencyKey(step), + idempotencyKey: backstopIdempotencyKey( + runId, + step + ), } ) ); @@ -1952,7 +1956,10 @@ export function workflowEntrypoint( requestedAt: new Date(), }, { - idempotencyKey: step.correlationId, + idempotencyKey: runScopedKey( + runId, + step.correlationId + ), } ) ); @@ -1968,6 +1975,7 @@ export function workflowEntrypoint( requestedAt: new Date(), }, getWaitContinuationDispatch( + runId, suspensionResult.waitTimeout.seconds, suspensionResult.waitTimeout.correlationId ) @@ -2508,7 +2516,7 @@ export function workflowEntrypoint( // correlationId so it dedupes against the // keyed re-dispatch the suspension handler // performs on replay (it also uses - // `idempotencyKey: step.correlationId`). + // the same run-scoped correlationId key). // // Without this, a mixed batch where one step // `completed` with unflushed background ops @@ -2527,7 +2535,10 @@ export function workflowEntrypoint( // retry body could run early/concurrently. // Sharing the key lets the earlier delayed // message win, honoring the backoff. - idempotencyKey: step.correlationId, + idempotencyKey: runScopedKey( + runId, + step.correlationId + ), } ) ) diff --git a/packages/core/src/runtime/idempotency-key.test.ts b/packages/core/src/runtime/idempotency-key.test.ts new file mode 100644 index 0000000000..c4b4ca5cbb --- /dev/null +++ b/packages/core/src/runtime/idempotency-key.test.ts @@ -0,0 +1,85 @@ +import { readdirSync, readFileSync } from 'node:fs'; +import { join, relative } from 'node:path'; +import { fileURLToPath } from 'node:url'; +import { describe, expect, it } from 'vitest'; +import { runScopedKey } from './idempotency-key.js'; + +const CORE_SRC = fileURLToPath(new URL('..', import.meta.url)); + +/** + * Expressions allowed as the value of an `idempotencyKey:` property in core: + * the two run-scoping builders, or a type declaration. + */ +const RUN_SCOPED_VALUE = + /^\s*(string;|runScopedKey\(|backstopIdempotencyKey\()/; + +function sourceFiles(dir: string): string[] { + return readdirSync(dir, { withFileTypes: true }).flatMap((entry) => { + const path = join(dir, entry.name); + if (entry.isDirectory()) return sourceFiles(path); + if (!entry.name.endsWith('.ts')) return []; + if (entry.name.endsWith('.test.ts')) return []; + return [path]; + }); +} + +/** Every place the file assigns an `idempotencyKey`, with the value assigned. */ +function keySites( + file: string, + lines: string[] +): Array<{ at: string; value: string }> { + const PROPERTY = 'idempotencyKey:'; + return lines.flatMap((line, index) => { + const at = line.indexOf(PROPERTY); + const trimmed = line.trim(); + if (at === -1 || trimmed.startsWith('//') || trimmed.startsWith('*')) { + return []; + } + // The value may sit on the following line when the formatter wraps it. + const sameLine = line.slice(at + PROPERTY.length); + return [ + { + at: `${relative(CORE_SRC, file)}:${index + 1}: ${trimmed}`, + value: sameLine.trim() ? sameLine : (lines[index + 1] ?? ''), + }, + ]; + }); +} + +describe('runScopedKey', () => { + it('prefixes the run and joins parts with colons', () => { + expect(runScopedKey('wrun_1', 'step_2', 'backstop')).toBe( + 'wrun_1:step_2:backstop' + ); + }); + + it('separates runs sharing a correlation id', () => { + // The whole point: under slot identity the first step of every run of a + // workflow is `step_…001`, and the queue those messages are sent to is + // shared by every run of that workflow. + expect(runScopedKey('wrun_a', 'step_001')).not.toBe( + runScopedKey('wrun_b', 'step_001') + ); + }); +}); + +describe('queue idempotency keys', () => { + /** + * A key that is not run-scoped is dropped silently by the world's dedupe for + * the length of its retention window (24h on Vercel Queues): the send is + * answered normally, no callback is dispatched, and the step is never + * executed. There is no error to find afterwards, so the only defence is that + * no site produces a key any other way. + */ + it('are all produced by the run-scoping builders', () => { + const sites = sourceFiles(CORE_SRC).flatMap((file) => + keySites(file, readFileSync(file, 'utf8').split('\n')) + ); + + expect(sites.filter((site) => !RUN_SCOPED_VALUE.test(site.value))).toEqual( + [] + ); + // Guard the scan itself: a pattern that matches nothing would pass above. + expect(sites.length).toBeGreaterThanOrEqual(4); + }); +}); diff --git a/packages/core/src/runtime/idempotency-key.ts b/packages/core/src/runtime/idempotency-key.ts new file mode 100644 index 0000000000..1a3ed9b1d9 --- /dev/null +++ b/packages/core/src/runtime/idempotency-key.ts @@ -0,0 +1,33 @@ +/** + * Run scoping for queue idempotency keys. + * + * Every keyed enqueue the runtime performs is deduped by the world on + * `(queue, idempotencyKey)`, and the dedupe record outlives the first delivery + * by a long way — Vercel Queues keeps it until message-retention TTL (24h), + * world-postgres keeps a completed-keys cache. Queue names are per workflow, not + * per run, so any key built only from a correlation id is shared by every + * concurrent run of that workflow the moment correlation ids stop being unique + * per run — which is exactly what slot numbering does: the first step of every + * run of `myWorkflow` is `step_…001`. + * + * A collision there is silent and total. The send is answered normally (Vercel + * Queues v3 returns a fresh message id; only the legacy provider reported + * duplicates), so no error surfaces in the SDK; the dispatcher records the + * message as a duplicate and excludes it from notifications, so no callback is + * dispatched; the orchestrator returns without a timeout and acks. The step is + * never executed, nothing anywhere reports a failure, and the run stalls for the + * length of the dedupe window. The inline-ownership backstop cannot recover it + * either — that path needs a `step_started` which never happened. + * + * So every key carries the run it belongs to, built here rather than at each + * call site. Keys are opaque to the worlds, so the prefix costs only length: a + * run id plus a correlation id plus a suffix is well under the 256-char cap. + */ + +/** + * Builds a queue idempotency key scoped to `runId`. Parts are joined with `:`, + * so a key keeps the readable shape it had before scoping was introduced. + */ +export function runScopedKey(runId: string, ...parts: string[]): string { + return [runId, ...parts].join(':'); +} diff --git a/packages/core/src/runtime/step-ownership.test.ts b/packages/core/src/runtime/step-ownership.test.ts index 5fda3bb4f2..c0de6cece9 100644 --- a/packages/core/src/runtime/step-ownership.test.ts +++ b/packages/core/src/runtime/step-ownership.test.ts @@ -1,6 +1,7 @@ import type { Event } from '@workflow/world'; import { afterEach, describe, expect, it } from 'vitest'; import type { StepInvocationQueueItem } from '../global.js'; +import { runScopedKey } from './idempotency-key.js'; import { backstopIdempotencyKey, hasPendingStepOwnedByMessage, @@ -9,6 +10,7 @@ import { } from './step-ownership.js'; const LEASE_ENV = 'WORKFLOW_INLINE_OWNERSHIP_LEASE_SECONDS'; +const RUN_ID = 'wrun_01ABC'; function makeStep( overrides: Partial = {} @@ -86,34 +88,47 @@ describe('stepLeaseRemainingSeconds', () => { }); describe('backstopIdempotencyKey', () => { - it('never collides with the step message dedupe key (bare correlationId)', () => { - // The owner's retry handoff enqueues the step keyed by correlationId - // with a ~1s backoff; a backstop occupying that key would absorb the - // retry and stall the run for the full lease. + it('never collides with the step message dedupe key', () => { + // The owner's retry handoff enqueues the step keyed by its run-scoped + // correlationId with a ~1s backoff; a backstop occupying that key would + // absorb the retry and stall the run for the full lease. const step = makeStep(); - expect(backstopIdempotencyKey(step)).not.toBe(step.correlationId); + expect(backstopIdempotencyKey(RUN_ID, step)).not.toBe( + runScopedKey(RUN_ID, step.correlationId) + ); + }); + + it('is isolated per run', () => { + // Slot-numbered correlation IDs repeat across runs of a workflow, and + // queues dedupe per queue — which is per workflow, not per run. + const step = makeStep(); + expect(backstopIdempotencyKey('wrun_other', step)).not.toBe( + backstopIdempotencyKey(RUN_ID, step) + ); }); it('is stable across wake replays within one ownership epoch', () => { // Every wake that observes the same latest step_started derives the // same key, capping fan-out at one pending backstop per epoch. - expect(backstopIdempotencyKey(makeStep())).toBe( - backstopIdempotencyKey(makeStep()) + expect(backstopIdempotencyKey(RUN_ID, makeStep())).toBe( + backstopIdempotencyKey(RUN_ID, makeStep()) ); }); it('changes when owner recovery re-stamps the step', () => { const initial = makeStep({ lastStartedAt: 1_000_000 }); const reStamped = makeStep({ lastStartedAt: 1_030_000 }); - expect(backstopIdempotencyKey(reStamped)).not.toBe( - backstopIdempotencyKey(initial) + expect(backstopIdempotencyKey(RUN_ID, reStamped)).not.toBe( + backstopIdempotencyKey(RUN_ID, initial) ); }); it('is isolated per correlation ID', () => { expect( - backstopIdempotencyKey(makeStep({ correlationId: 'step_A' })) - ).not.toBe(backstopIdempotencyKey(makeStep({ correlationId: 'step_B' }))); + backstopIdempotencyKey(RUN_ID, makeStep({ correlationId: 'step_A' })) + ).not.toBe( + backstopIdempotencyKey(RUN_ID, makeStep({ correlationId: 'step_B' })) + ); }); it('re-arms through a full owner-recovery cycle despite in-flight key retention', () => { @@ -142,9 +157,9 @@ describe('backstopIdempotencyKey', () => { // Epoch 1: owner stamps at T0; a wake replay arms the backstop. const epoch1 = makeStep({ lastStartedAt: 1_000_000 }); - expect(enqueue(backstopIdempotencyKey(epoch1))).toBe('accepted'); + expect(enqueue(backstopIdempotencyKey(RUN_ID, epoch1))).toBe('accepted'); // A second wake in the same epoch is deduped (fan-out stays capped). - expect(enqueue(backstopIdempotencyKey(epoch1))).toBe('deduped'); + expect(enqueue(backstopIdempotencyKey(RUN_ID, epoch1))).toBe('deduped'); // Owner crashes; queue redelivery re-stamps step_started at T1 // (owner recovery) → new ownership epoch, lease refreshed. @@ -153,7 +168,7 @@ describe('backstopIdempotencyKey', () => { // The epoch-1 backstop fires during the refreshed lease. Its handler // replays, sees ownership active with time remaining, and re-arms — // while its own message is still in flight (key not yet released). - const rearm = enqueue(backstopIdempotencyKey(epoch2)); + const rearm = enqueue(backstopIdempotencyKey(RUN_ID, epoch2)); expect(rearm).toBe('accepted'); // Owner dies for good: the epoch-2 backstop is the recovery path, and diff --git a/packages/core/src/runtime/step-ownership.ts b/packages/core/src/runtime/step-ownership.ts index 021511a175..b05d49cd19 100644 --- a/packages/core/src/runtime/step-ownership.ts +++ b/packages/core/src/runtime/step-ownership.ts @@ -1,6 +1,7 @@ import type { Event } from '@workflow/world'; import type { StepInvocationQueueItem } from '../global.js'; import { getInlineOwnershipLeaseSeconds } from './constants.js'; +import { runScopedKey } from './idempotency-key.js'; /** * Inline step ownership helpers for the pending-step dispatch decision table @@ -73,12 +74,23 @@ export function stepLeaseRemainingSeconds( * lease has time remaining, which requires `lastStartedAt` to be set. * * The key must also never be the step message's own `idempotencyKey` - * (the bare correlation ID): the owner's retry handoff enqueues the step - * under that key with a short backoff, and a pending backstop sharing it - * would absorb the retry — turning a 1s backoff into a full-lease stall. + * (the run-scoped correlation ID): the owner's retry handoff enqueues the + * step under that key with a short backoff, and a pending backstop sharing + * it would absorb the retry — turning a 1s backoff into a full-lease stall. + * + * Scoped to the run because correlation IDs are only unique within one (see + * `idempotency-key.ts`). */ -export function backstopIdempotencyKey(step: StepInvocationQueueItem): string { - return `${step.correlationId}:backstop:${step.lastStartedAt}`; +export function backstopIdempotencyKey( + runId: string, + step: StepInvocationQueueItem +): string { + return runScopedKey( + runId, + step.correlationId, + 'backstop', + String(step.lastStartedAt) + ); } /** diff --git a/packages/core/src/runtime/wait-continuation.test.ts b/packages/core/src/runtime/wait-continuation.test.ts index 3d79cd5b91..dc611a9524 100644 --- a/packages/core/src/runtime/wait-continuation.test.ts +++ b/packages/core/src/runtime/wait-continuation.test.ts @@ -5,39 +5,57 @@ import { WAIT_CONTINUATION_MAX_DELAY_SECONDS, } from './wait-continuation.js'; +const RUN_ID = 'wrun_01ABC'; const CORR_ID = 'wait_01ABC'; +/** The run-scoped key a mid-range wait dedupes on. */ +const KEY = `${RUN_ID}:${CORR_ID}`; const NOW = new Date('2026-05-19T12:00:20.500Z').getTime(); describe('getWaitContinuationDispatch', () => { - describe('mid-range waits (bare correlationId key)', () => { - it('uses the bare correlationId so re-observations dedupe', () => { - expect(getWaitContinuationDispatch(60, CORR_ID, NOW)).toEqual({ + describe('mid-range waits (unsuffixed key)', () => { + it('uses the run-scoped correlationId so re-observations dedupe', () => { + expect(getWaitContinuationDispatch(RUN_ID, 60, CORR_ID, NOW)).toEqual({ delaySeconds: 60, - idempotencyKey: CORR_ID, + idempotencyKey: KEY, }); }); + it('is isolated per run', () => { + // Slot-numbered correlation IDs repeat across runs of a workflow, and a + // queue's dedupe scope is the workflow, not the run — so two runs + // sleeping at the same point must not share a continuation key. + const other = getWaitContinuationDispatch('wrun_other', 60, CORR_ID, NOW); + expect(other.idempotencyKey).not.toBe(KEY); + }); + it('is stable across suspension passes targeting the same deadline', () => { - const pass1 = getWaitContinuationDispatch(60, CORR_ID, NOW); - const pass2 = getWaitContinuationDispatch(45, CORR_ID, NOW + 15_000); + const pass1 = getWaitContinuationDispatch(RUN_ID, 60, CORR_ID, NOW); + const pass2 = getWaitContinuationDispatch( + RUN_ID, + 45, + CORR_ID, + NOW + 15_000 + ); expect(pass2.idempotencyKey).toBe(pass1.idempotencyKey); }); it('covers the full band up to the max delay', () => { const low = getWaitContinuationDispatch( + RUN_ID, NEAR_ELAPSED_WAIT_THRESHOLD_SECONDS + 1, CORR_ID, NOW ); const high = getWaitContinuationDispatch( + RUN_ID, WAIT_CONTINUATION_MAX_DELAY_SECONDS, CORR_ID, NOW ); - expect(low.idempotencyKey).toBe(CORR_ID); + expect(low.idempotencyKey).toBe(KEY); expect(high).toEqual({ delaySeconds: WAIT_CONTINUATION_MAX_DELAY_SECONDS, - idempotencyKey: CORR_ID, + idempotencyKey: KEY, }); }); }); @@ -46,22 +64,28 @@ describe('getWaitContinuationDispatch', () => { it('suffixes the key with the current epoch second', () => { expect( getWaitContinuationDispatch( + RUN_ID, NEAR_ELAPSED_WAIT_THRESHOLD_SECONDS, CORR_ID, NOW ) ).toEqual({ delaySeconds: NEAR_ELAPSED_WAIT_THRESHOLD_SECONDS, - idempotencyKey: `${CORR_ID}:${Math.floor(NOW / 1000)}`, + idempotencyKey: `${KEY}:${Math.floor(NOW / 1000)}`, }); }); it('collapses same-second duplicates but frees the key for a later retry', () => { - const first = getWaitContinuationDispatch(1, CORR_ID, NOW); - const sameSecond = getWaitContinuationDispatch(1, CORR_ID, NOW + 400); + const first = getWaitContinuationDispatch(RUN_ID, 1, CORR_ID, NOW); + const sameSecond = getWaitContinuationDispatch( + RUN_ID, + 1, + CORR_ID, + NOW + 400 + ); // A retry can only be enqueued after the >= 1s delay of the first // message, which guarantees a later epoch-second bucket. - const retry = getWaitContinuationDispatch(1, CORR_ID, NOW + 1000); + const retry = getWaitContinuationDispatch(RUN_ID, 1, CORR_ID, NOW + 1000); expect(sameSecond.idempotencyKey).toBe(first.idempotencyKey); expect(retry.idempotencyKey).not.toBe(first.idempotencyKey); }); @@ -71,15 +95,23 @@ describe('getWaitContinuationDispatch', () => { const SEVEN_DAYS = 7 * 24 * 3600; // 604800s > 7 * MAX_DELAY (579600s) it('clamps the delay to the max and suffixes the key with the hop index', () => { - expect(getWaitContinuationDispatch(SEVEN_DAYS, CORR_ID, NOW)).toEqual({ + expect( + getWaitContinuationDispatch(RUN_ID, SEVEN_DAYS, CORR_ID, NOW) + ).toEqual({ delaySeconds: WAIT_CONTINUATION_MAX_DELAY_SECONDS, - idempotencyKey: `${CORR_ID}:hop-8`, + idempotencyKey: `${KEY}:hop-8`, }); }); it('keeps the key stable for re-observations within the same hop window', () => { - const pass1 = getWaitContinuationDispatch(SEVEN_DAYS, CORR_ID, NOW); + const pass1 = getWaitContinuationDispatch( + RUN_ID, + SEVEN_DAYS, + CORR_ID, + NOW + ); const pass2 = getWaitContinuationDispatch( + RUN_ID, SEVEN_DAYS - 3600, CORR_ID, NOW + 3600_000 @@ -92,6 +124,7 @@ describe('getWaitContinuationDispatch', () => { const keys: string[] = []; while (remaining > NEAR_ELAPSED_WAIT_THRESHOLD_SECONDS) { const { delaySeconds, idempotencyKey } = getWaitContinuationDispatch( + RUN_ID, remaining, CORR_ID, NOW + (SEVEN_DAYS - remaining) * 1000 @@ -107,18 +140,19 @@ describe('getWaitContinuationDispatch', () => { expect(new Set(keys).size).toBe(keys.length); // 604800s chains as 7 max-delay hops + 1 remainder hop. expect(keys).toHaveLength(8); - expect(keys[keys.length - 1]).toBe(CORR_ID); + expect(keys[keys.length - 1]).toBe(KEY); }); it('uses a fresh key when the final partial hop lands in the near-elapsed band', () => { // Remaining drops below the near-elapsed threshold only at the very // end; the second-bucketed key never collides with hop keys. const nearEnd = getWaitContinuationDispatch( + RUN_ID, NEAR_ELAPSED_WAIT_THRESHOLD_SECONDS, CORR_ID, NOW + SEVEN_DAYS * 1000 ); - expect(nearEnd.idempotencyKey).toMatch(new RegExp(`^${CORR_ID}:\\d+$`)); + expect(nearEnd.idempotencyKey).toMatch(new RegExp(`^${KEY}:\\d+$`)); }); }); @@ -142,6 +176,7 @@ describe('getWaitContinuationDispatch', () => { // A wait exactly at the default near-elapsed threshold would previously // return its full (> max) remaining time as the delay. const { delaySeconds } = getWaitContinuationDispatch( + RUN_ID, NEAR_ELAPSED_WAIT_THRESHOLD_SECONDS, CORR_ID, NOW diff --git a/packages/core/src/runtime/wait-continuation.ts b/packages/core/src/runtime/wait-continuation.ts index 02eb1cffc8..97790c50c5 100644 --- a/packages/core/src/runtime/wait-continuation.ts +++ b/packages/core/src/runtime/wait-continuation.ts @@ -7,6 +7,9 @@ * elapsed waits" pass). This module decides the message's `delaySeconds` * and `idempotencyKey`. * + * Every key is scoped to the run (see `idempotency-key.ts`); the variations + * below are suffixes on top of that. + * * The continuation is keyed on the wait's correlationId: while a wait is * pending, every replay pass over the run re-observes it (e.g., once per * step completion in `Promise.all([steps..., sleep()])`), and without @@ -58,6 +61,7 @@ */ import { envNumber } from '@workflow/world'; +import { runScopedKey } from './idempotency-key.js'; /** * Maximum `delaySeconds` for a single wait-continuation message. Waits @@ -99,9 +103,11 @@ export interface WaitContinuationDispatch { * Computes the queue delay and idempotency key for a wait-continuation * message. `timeoutSeconds` is the time until the wait's `resumeAt` * (floored at 1s by the suspension handler); `waitCorrelationId` - * identifies the wait so repeated suspension passes dedupe. + * identifies the wait so repeated suspension passes dedupe, and `runId` + * keeps it from deduping against another run of the same workflow. */ export function getWaitContinuationDispatch( + runId: string, timeoutSeconds: number, waitCorrelationId: string, now: number = Date.now() @@ -120,14 +126,20 @@ export function getWaitContinuationDispatch( if (timeoutSeconds <= nearElapsedThreshold) { return { delaySeconds: timeoutSeconds, - idempotencyKey: `${waitCorrelationId}:${Math.floor(now / 1000)}`, + idempotencyKey: runScopedKey( + runId, + waitCorrelationId, + String(Math.floor(now / 1000)) + ), }; } const hop = Math.ceil(timeoutSeconds / maxDelaySeconds); + // First hop carries no suffix, so a single-hop wait keeps exactly one key for + // its lifetime. + const hopSuffix = hop === 1 ? [] : [`hop-${hop}`]; return { delaySeconds: Math.min(timeoutSeconds, maxDelaySeconds), - idempotencyKey: - hop === 1 ? waitCorrelationId : `${waitCorrelationId}:hop-${hop}`, + idempotencyKey: runScopedKey(runId, waitCorrelationId, ...hopSuffix), }; } diff --git a/packages/core/src/step-delivery-hop-count.test.ts b/packages/core/src/step-delivery-hop-count.test.ts index f7ea3e09b1..1ca6d9ab90 100644 --- a/packages/core/src/step-delivery-hop-count.test.ts +++ b/packages/core/src/step-delivery-hop-count.test.ts @@ -29,6 +29,7 @@ import type { Event } from '@workflow/world'; import * as nanoid from 'nanoid'; import { monotonicFactory } from 'ulid'; import { describe, expect, it, vi } from 'vitest'; +import { createCorrelationIdFactory } from './correlation-ids.js'; import { EventsConsumer } from './events-consumer.js'; import { WorkflowSuspension } from './global.js'; import type { WorkflowOrchestratorContext } from './private.js'; @@ -69,6 +70,10 @@ function setupWorkflowContext( }), invocationsQueue: new Map(), generateUlid: () => ulid(workflowStartedAt), + nextCorrelationId: createCorrelationIdFactory({ + specVersion: undefined, + generateUlid: () => ulid(workflowStartedAt), + }), generateNanoid: nanoid.customRandom(nanoid.urlAlphabet, 21, (size) => new Uint8Array(size).map(() => 256 * context.globalThis.Math.random()) ), diff --git a/packages/core/src/step-delivery-ordering.test.ts b/packages/core/src/step-delivery-ordering.test.ts index 85933b7e7c..1f52e092f7 100644 --- a/packages/core/src/step-delivery-ordering.test.ts +++ b/packages/core/src/step-delivery-ordering.test.ts @@ -4,6 +4,7 @@ import type { Event } from '@workflow/world'; import * as nanoid from 'nanoid'; import { monotonicFactory } from 'ulid'; import { afterEach, describe, expect, it, vi } from 'vitest'; +import { createCorrelationIdFactory } from './correlation-ids.js'; import { EventsConsumer } from './events-consumer.js'; import { WorkflowSuspension } from './global.js'; import type { WorkflowOrchestratorContext } from './private.js'; @@ -118,6 +119,10 @@ function setupWorkflowContext( }), invocationsQueue: new Map(), generateUlid: () => ulid(workflowStartedAt), + nextCorrelationId: createCorrelationIdFactory({ + specVersion: undefined, + generateUlid: () => ulid(workflowStartedAt), + }), generateNanoid: nanoid.customRandom(nanoid.urlAlphabet, 21, (size) => new Uint8Array(size).map(() => 256 * context.globalThis.Math.random()) ), diff --git a/packages/core/src/step-hydration-memoization.test.ts b/packages/core/src/step-hydration-memoization.test.ts index 7df9abdf38..97bc5687e4 100644 --- a/packages/core/src/step-hydration-memoization.test.ts +++ b/packages/core/src/step-hydration-memoization.test.ts @@ -2,6 +2,7 @@ import type { Event } from '@workflow/world'; import * as nanoid from 'nanoid'; import { monotonicFactory } from 'ulid'; import { afterEach, describe, expect, it, vi } from 'vitest'; +import { createCorrelationIdFactory } from './correlation-ids.js'; import { EventsConsumer } from './events-consumer.js'; import type { WorkflowOrchestratorContext } from './private.js'; import { ReplayPayloadCache } from './replay-payload-cache.js'; @@ -43,6 +44,10 @@ function setupWorkflowContext( }), invocationsQueue: new Map(), generateUlid: () => ulid(workflowStartedAt), + nextCorrelationId: createCorrelationIdFactory({ + specVersion: undefined, + generateUlid: () => ulid(workflowStartedAt), + }), generateNanoid: nanoid.customRandom(nanoid.urlAlphabet, 21, (size) => new Uint8Array(size).map(() => 256 * context.globalThis.Math.random()) ), diff --git a/packages/core/src/step.test.ts b/packages/core/src/step.test.ts index 15540cc875..4fdd2dfe63 100644 --- a/packages/core/src/step.test.ts +++ b/packages/core/src/step.test.ts @@ -9,6 +9,7 @@ import * as nanoid from 'nanoid'; import { monotonicFactory } from 'ulid'; import { beforeAll, describe, expect, it, vi } from 'vitest'; import { registerSerializationClass } from './class-serialization.js'; +import { createCorrelationIdFactory } from './correlation-ids.js'; import { EventsConsumer } from './events-consumer.js'; import { WorkflowSuspension } from './global.js'; import type { WorkflowOrchestratorContext } from './private.js'; @@ -62,6 +63,10 @@ function setupWorkflowContext(events: Event[]): WorkflowOrchestratorContext { }), invocationsQueue: new Map(), generateUlid: () => ulid(workflowStartedAt), // All generated ulids use the workflow's started at time + nextCorrelationId: createCorrelationIdFactory({ + specVersion: undefined, + generateUlid: () => ulid(workflowStartedAt), + }), generateNanoid: nanoid.customRandom(nanoid.urlAlphabet, 21, (size) => new Uint8Array(size).map(() => 256 * context.globalThis.Math.random()) ), diff --git a/packages/core/src/step.ts b/packages/core/src/step.ts index 7801b25d6d..0deb4e5c64 100644 --- a/packages/core/src/step.ts +++ b/packages/core/src/step.ts @@ -24,7 +24,7 @@ export function createUseStep(ctx: WorkflowOrchestratorContext) { ): Promise { const { promise, resolve, reject } = withResolvers(); - const correlationId = `step_${ctx.generateUlid()}`; + const correlationId = ctx.nextCorrelationId('step'); const queueItem: StepInvocationQueueItem = { type: 'step', diff --git a/packages/core/src/workflow.ts b/packages/core/src/workflow.ts index 3f73899687..fde4b70250 100644 --- a/packages/core/src/workflow.ts +++ b/packages/core/src/workflow.ts @@ -10,7 +10,7 @@ import type { Event, WorkflowRun, WorldCapabilities } from '@workflow/world'; import { SPEC_VERSION_SUPPORTS_COMPRESSION } from '@workflow/world'; import * as nanoid from 'nanoid'; import { monotonicFactory } from 'ulid'; -import type { PayloadKey } from './serialization/encryption.js'; +import { createCorrelationIdFactory } from './correlation-ids.js'; import { EventConsumerResult, EventsConsumer } from './events-consumer.js'; import type { QueueItem } from './global.js'; import { ENOTSUP, WorkflowSuspension } from './global.js'; @@ -21,6 +21,7 @@ import { getPortLazy } from './runtime/get-port-lazy.js'; import { runIdCreatedAt } from './runtime/run-id-time.js'; import { handleSuspension } from './runtime/suspension-handler.js'; import { getWorld } from './runtime/world.js'; +import type { PayloadKey } from './serialization/encryption.js'; import { dehydrateWorkflowReturnValue, hydrateWorkflowArguments, @@ -216,6 +217,13 @@ export async function runWorkflow( // Correlation IDs must be replay-stable. `startedAt` differs between a // turbo delivery and a later server-backed replay, so use fixedTimestamp. generateUlid: () => ulid(fixedTimestamp), + // Mode comes from the run's persisted spec version, never from this + // build: a run whose log holds ULID correlation ids must keep proposing + // them however new the code replaying it is. + nextCorrelationId: createCorrelationIdFactory({ + specVersion: workflowRun.specVersion, + generateUlid: () => ulid(fixedTimestamp), + }), generateNanoid, invocationsQueue: new Map(), // Use getter/setter so the EventsConsumer's getPromiseQueue() always diff --git a/packages/core/src/workflow/hook.test.ts b/packages/core/src/workflow/hook.test.ts index f6a864ae15..86cf8b54a3 100644 --- a/packages/core/src/workflow/hook.test.ts +++ b/packages/core/src/workflow/hook.test.ts @@ -12,6 +12,7 @@ import { aliasSerializationClass, RUN_CLASS_ID, } from '../class-serialization.js'; +import { createCorrelationIdFactory } from '../correlation-ids.js'; import { EventsConsumer } from '../events-consumer.js'; import { WorkflowSuspension } from '../global.js'; import type { WorkflowOrchestratorContext } from '../private.js'; @@ -47,6 +48,10 @@ function setupWorkflowContext(events: Event[]): WorkflowOrchestratorContext { }), invocationsQueue: new Map(), generateUlid: () => ulid(workflowStartedAt), + nextCorrelationId: createCorrelationIdFactory({ + specVersion: undefined, + generateUlid: () => ulid(workflowStartedAt), + }), generateNanoid: nanoid.customRandom(nanoid.urlAlphabet, 21, (size) => new Uint8Array(size).map(() => 256 * context.globalThis.Math.random()) ), diff --git a/packages/core/src/workflow/sleep.test.ts b/packages/core/src/workflow/sleep.test.ts index 59d9ca8b91..6062a85981 100644 --- a/packages/core/src/workflow/sleep.test.ts +++ b/packages/core/src/workflow/sleep.test.ts @@ -4,6 +4,7 @@ import type { Event } from '@workflow/world'; import * as nanoid from 'nanoid'; import { monotonicFactory } from 'ulid'; import { describe, expect, it, vi } from 'vitest'; +import { createCorrelationIdFactory } from '../correlation-ids.js'; import { EventsConsumer } from '../events-consumer.js'; import { WorkflowSuspension } from '../global.js'; import type { WorkflowOrchestratorContext } from '../private.js'; @@ -38,6 +39,10 @@ function setupWorkflowContext(events: Event[]): WorkflowOrchestratorContext { }), invocationsQueue: new Map(), generateUlid: () => ulid(workflowStartedAt), + nextCorrelationId: createCorrelationIdFactory({ + specVersion: undefined, + generateUlid: () => ulid(workflowStartedAt), + }), generateNanoid: nanoid.customRandom(nanoid.urlAlphabet, 21, (size) => new Uint8Array(size).map(() => 256 * context.globalThis.Math.random()) ), diff --git a/packages/core/src/workflow/sleep.ts b/packages/core/src/workflow/sleep.ts index c8848d0c3a..aedf1d2392 100644 --- a/packages/core/src/workflow/sleep.ts +++ b/packages/core/src/workflow/sleep.ts @@ -15,7 +15,7 @@ export function createSleep(ctx: WorkflowOrchestratorContext) { param: StringValue | Date | number ): Promise { const { promise, resolve } = withResolvers(); - const correlationId = `wait_${ctx.generateUlid()}`; + const correlationId = ctx.nextCorrelationId('wait'); // Calculate the resume time const resumeAt = parseDurationToDate(param); diff --git a/packages/world/package.json b/packages/world/package.json index 681c1d3327..84d5f1b1a0 100644 --- a/packages/world/package.json +++ b/packages/world/package.json @@ -20,6 +20,7 @@ "scripts": { "build": "tsc", "dev": "tsc --watch", + "test": "vitest run src", "clean": "tsc --build --clean && rm -rf dist" }, "dependencies": { diff --git a/packages/world/src/index.ts b/packages/world/src/index.ts index 3d13620de8..c6f601f36c 100644 --- a/packages/world/src/index.ts +++ b/packages/world/src/index.ts @@ -101,16 +101,25 @@ export { PaginatedResponseSchema, StructuredErrorSchema, } from './shared.js'; +export { + FIRST_SLOT, + isSlotId, + SLOT_ID_WIDTH, + slotFromId, + slotIdBody, +} from './slot-identity.js'; export type { SpecVersion } from './spec-version.js'; export { isLegacySpecVersion, requiresNewerWorld, SPEC_VERSION_CURRENT, SPEC_VERSION_LEGACY, + SPEC_VERSION_SLOT_IDENTITY, SPEC_VERSION_SUPPORTS_ATTRIBUTES, SPEC_VERSION_SUPPORTS_CBOR_QUEUE_TRANSPORT, SPEC_VERSION_SUPPORTS_COMPRESSION, SPEC_VERSION_SUPPORTS_EVENT_SOURCING, + usesSlotIdentity, } from './spec-version.js'; export type * from './steps.js'; export { diff --git a/packages/world/src/slot-identity.test.ts b/packages/world/src/slot-identity.test.ts new file mode 100644 index 0000000000..46c5e7d7d7 --- /dev/null +++ b/packages/world/src/slot-identity.test.ts @@ -0,0 +1,59 @@ +import { ulid } from 'ulid'; +import { describe, expect, it } from 'vitest'; +import { + FIRST_SLOT, + isSlotId, + SLOT_ID_WIDTH, + slotFromId, + slotIdBody, +} from './slot-identity.js'; +import { ulidToDate } from './ulid.js'; + +describe('slotIdBody', () => { + it('pads to ULID width so a slot is accepted wherever a ULID is', () => { + const body = slotIdBody(FIRST_SLOT); + expect(body).toHaveLength(SLOT_ID_WIDTH); + expect(`evnt_${body}`).toHaveLength(`evnt_${ulid()}`.length); + // Crockford base32 starts with the decimal digits, so the padded body + // parses as a ULID — this is what keeps every existing schema, sort key and + // range fence working unchanged. + expect(ulidToDate(body)).not.toBeNull(); + }); + + it('orders lexicographically by slot at a fixed width', () => { + const ascending = [1, 2, 9, 10, 100].map(slotIdBody); + expect([...ascending].sort()).toEqual(ascending); + }); + + it('rejects slots outside the dense numbering', () => { + expect(() => slotIdBody(0)).toThrow(); + expect(() => slotIdBody(-1)).toThrow(); + expect(() => slotIdBody(1.5)).toThrow(); + }); +}); + +describe('slotFromId', () => { + it('round-trips a prefixed id', () => { + expect(slotFromId(`step_${slotIdBody(42)}`)).toBe(42); + }); + + it('round-trips a bare body', () => { + expect(slotFromId(slotIdBody(42))).toBe(42); + }); + + it('reads no slot out of a ULID id', () => { + expect(slotFromId(`evnt_${ulid()}`)).toBeUndefined(); + expect(isSlotId(`evnt_${ulid()}`)).toBe(false); + }); + + it('reads no slot out of the all-zero range fence', () => { + // Slot 0 is the inclusive lower fence for range queries over a run's + // events, never an event. + expect(slotFromId('0'.repeat(SLOT_ID_WIDTH))).toBeUndefined(); + }); + + it('reads no slot out of a body of the wrong width', () => { + expect(slotFromId('evnt_1')).toBeUndefined(); + expect(slotFromId(`evnt_${'1'.repeat(SLOT_ID_WIDTH + 1)}`)).toBeUndefined(); + }); +}); diff --git a/packages/world/src/slot-identity.ts b/packages/world/src/slot-identity.ts new file mode 100644 index 0000000000..c4d609c754 --- /dev/null +++ b/packages/world/src/slot-identity.ts @@ -0,0 +1,63 @@ +/** + * Slot identity: dense, per-run numbering for event ids and correlation ids. + * + * A slot id's body is a decimal counter zero-padded to ULID width. Crockford's + * base32 alphabet begins with the ten decimal digits, so that body is a + * syntactically valid ULID body — every schema, sort key, range fence and + * cursor that accepted a ULID keeps accepting a slot — and because the width is + * fixed, lexicographic order is numeric order. + * + * Slots are dense and start at 1. Density is the whole point of the scheme: it + * makes `events.length === maxSlot` a proof that a loaded log has no holes, + * which is something a server-minted ULID log can never offer. Zero is left + * unused because the inclusive lower fence for range queries over a run's + * events is the all-zero id. + * + * A slot body decodes as a ULID *timestamp* of epoch 0 without erroring, so + * nothing may read a time out of one. Use the event's own `createdAt` / + * `occurredAt`. + */ + +/** Width of a slot id's body: ULID width, so a slot is accepted wherever a ULID is. */ +export const SLOT_ID_WIDTH = 26; + +/** First slot in a run. Slot 0 is unused — it is the inclusive range fence. */ +export const FIRST_SLOT = 1; + +const SLOT_BODY_PATTERN = new RegExp(`^[0-9]{${SLOT_ID_WIDTH}}$`); + +/** + * The id body naming `slot`, e.g. `1` → `00000000000000000000000001`. Callers + * prepend their own prefix (`evnt_`, `step_`, `wait_`). + */ +export function slotIdBody(slot: number): string { + if (!Number.isInteger(slot) || slot < FIRST_SLOT) { + throw new Error( + `Slot must be an integer >= ${FIRST_SLOT}, received ${slot}` + ); + } + const body = String(slot).padStart(SLOT_ID_WIDTH, '0'); + if (body.length > SLOT_ID_WIDTH) { + throw new Error(`Slot ${slot} does not fit in ${SLOT_ID_WIDTH} digits`); + } + return body; +} + +/** + * The slot named by an id, or undefined if the id is not a slot id. Accepts + * both a prefixed id (`step_0…001`) and a bare body. + */ +export function slotFromId(id: string): number | undefined { + const underscore = id.indexOf('_'); + const body = underscore === -1 ? id : id.slice(underscore + 1); + if (!SLOT_BODY_PATTERN.test(body)) { + return undefined; + } + const slot = Number(body); + return slot >= FIRST_SLOT ? slot : undefined; +} + +/** Whether an id numbers itself by slot rather than by ULID. */ +export function isSlotId(id: string): boolean { + return slotFromId(id) !== undefined; +} diff --git a/packages/world/src/spec-version.ts b/packages/world/src/spec-version.ts index cf516b772b..d9be743be6 100644 --- a/packages/world/src/spec-version.ts +++ b/packages/world/src/spec-version.ts @@ -31,6 +31,24 @@ export const SPEC_VERSION_SUPPORTS_ATTRIBUTES = 4 as SpecVersion; */ export const SPEC_VERSION_SUPPORTS_COMPRESSION = 5 as SpecVersion; +/** + * Runs at this spec version or later number their events, and their step and + * wait correlation ids, by dense per-run slots (`evnt_…001`, `step_…001`) + * instead of ULIDs. Ids are minted by the client, and a write that proposes an + * event id already taken is rejected rather than renumbered — which is what + * lets a client prove its loaded log is complete. + * + * A run is in exactly one mode for life: the mode is read from the run's + * persisted `specVersion`, never from the build. A run started under ULID + * correlation ids and replayed by a slot-capable build would otherwise propose + * `step_…001` where its log holds `step_01K…`, matching no existing entity. + * + * Deliberately not `SPEC_VERSION_CURRENT` yet: `requiresNewerWorld()` is what + * makes a world reject runs it cannot read, so bumping current before the + * worlds can allocate slots would have them reject their own new runs. + */ +export const SPEC_VERSION_SLOT_IDENTITY = 6 as SpecVersion; + /** * Current spec version (event-sourced architecture with native attributes * and compressed payloads). @@ -65,3 +83,15 @@ export function requiresNewerWorld(v: number | undefined | null): boolean { if (v === undefined || v === null) return false; return v > SPEC_VERSION_CURRENT; } + +/** + * Whether a run numbers its events and correlation ids by slot. Always pass the + * run's persisted `specVersion`; see `SPEC_VERSION_SLOT_IDENTITY`. + * + * @param v - The spec version number, or undefined/null for legacy runs + * @returns true if the run uses slot identity + */ +export function usesSlotIdentity(v: number | undefined | null): boolean { + if (v === undefined || v === null) return false; + return v >= SPEC_VERSION_SLOT_IDENTITY; +} From bb04d1c5938cdcf75e79685ffb06df5cf1422149 Mon Sep 17 00:00:00 2001 From: Peter Wielander Date: Thu, 30 Jul 2026 14:15:36 -0700 Subject: [PATCH 02/35] temp: point e2e at the slot-identity backend branch (revert before merge) --- packages/world-vercel/src/utils.ts | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/packages/world-vercel/src/utils.ts b/packages/world-vercel/src/utils.ts index 60157de448..efe9ba0fca 100644 --- a/packages/world-vercel/src/utils.ts +++ b/packages/world-vercel/src/utils.ts @@ -32,7 +32,10 @@ import { version } from './version.js'; * `main` — rewritten by external CI for branch-deployment testing. * Prefer `VERCEL_WORKFLOW_SERVER_URL` for deployment-time configuration. */ -export const WORKFLOW_SERVER_URL_OVERRIDE = ''; +// TEMPORARY — revert to '' before merge. Points e2e at the slot-identity +// backend branch deployment (top of the paired backend stack). +export const WORKFLOW_SERVER_URL_OVERRIDE = + 'https://workflow-server-git-peter-slot-ids-6-guards.vercel.sh'; /** * HTTP methods that are safe to transparently re-issue inside the adapter. From 3ae14e7bf7812bd9e21ac6ee00ff6b2e6128f369 Mon Sep 17 00:00:00 2001 From: Peter Wielander Date: Thu, 30 Jul 2026 14:16:19 -0700 Subject: [PATCH 03/35] test(world-vercel): honor the server URL override in the remaining v4 mocks --- packages/world-vercel/src/events-v4.test.ts | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/packages/world-vercel/src/events-v4.test.ts b/packages/world-vercel/src/events-v4.test.ts index 24746c2a30..a22e3fc12c 100644 --- a/packages/world-vercel/src/events-v4.test.ts +++ b/packages/world-vercel/src/events-v4.test.ts @@ -271,7 +271,8 @@ describe('getWorkflowRunEventsV4 over HTTP', () => { */ describe('getEventV4 over HTTP', () => { it('returns the first frame and stops reading the rest', async () => { - const origin = 'https://vercel-workflow.com'; + const origin = + WORKFLOW_SERVER_URL_OVERRIDE || 'https://vercel-workflow.com'; const agent = new MockAgent(); agent.disableNetConnect(); @@ -508,7 +509,8 @@ describe('createWorkflowRunEventV4 over HTTP', () => { }); it('forwards stateUpdatedAt in the frame meta (precondition guard)', async () => { - const origin = 'https://vercel-workflow.com'; + const origin = + WORKFLOW_SERVER_URL_OVERRIDE || 'https://vercel-workflow.com'; const agent = new MockAgent(); agent.disableNetConnect(); @@ -560,7 +562,8 @@ describe('createWorkflowRunEventV4 over HTTP', () => { }); it('omits stateUpdatedAt from the frame meta when not set', async () => { - const origin = 'https://vercel-workflow.com'; + const origin = + WORKFLOW_SERVER_URL_OVERRIDE || 'https://vercel-workflow.com'; const agent = new MockAgent(); agent.disableNetConnect(); From 59dc7a64e0972af5fc3a6fd44185abb1d6b4e0ff Mon Sep 17 00:00:00 2001 From: Peter Wielander Date: Thu, 30 Jul 2026 15:02:54 -0700 Subject: [PATCH 04/35] feat(core): claim event slots client-side and reclaim on conflict MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit On a run that numbers its events by slot, the runtime names each event's own id, which is its claim on that position in the log. The backend inserts the id conditionally, so a 409 proves another writer got there first and that this replay ran against a log missing at least one event. Claims are reserved contiguously off the loaded log rather than all at maxSlot + 1: a suspension flushes its operations concurrently, so without reservation every operation in a flush would propose the same slot and all but one would conflict, on every flush. Operations are built in deterministic replay order, so each one's slot is replay-stable. `withEventCreateFence` picks the run's fence: the event slot for a slot-numbered run, the `stateUpdatedAt` watermark otherwise. The two retry loops stay separate — they differ in what a rejection proves and in what the client does about it, and both are live at once while runs on the older numbering drain, which is what keeps 409s and 412s separately countable during a rollout. Creates that must not retry in place (`run_completed`, the inline `step_started` claims) take a bare fence from `eventCreateFenceFor`, so a rejection escapes to a fresh replay: merged events can change what the workflow body decides, and only a replay from the top can act on them. Co-Authored-By: Claude Opus 5 --- .changeset/slot-event-identity-client.md | 9 + .../api-reference/workflow-errors/index.mdx | 3 + .../api-reference/workflow-errors/meta.json | 1 + .../workflow-errors/slot-conflict-error.mdx | 76 ++++ packages/core/src/runtime.ts | 94 +++-- packages/core/src/runtime/helpers.test.ts | 343 ++++++++++++++++-- packages/core/src/runtime/helpers.ts | 204 ++++++++++- packages/core/src/runtime/step-executor.ts | 52 ++- .../core/src/runtime/suspension-handler.ts | 26 +- packages/errors/src/index.ts | 53 +++ packages/workflow/src/internal/errors.ts | 1 + packages/world-vercel/src/event-retry.test.ts | 11 + packages/world-vercel/src/event-retry.ts | 5 + packages/world-vercel/src/events-v4.test.ts | 224 +++++++++++- packages/world-vercel/src/events-v4.ts | 134 ++++++- packages/world-vercel/src/events.ts | 50 ++- packages/world/src/events.ts | 31 ++ packages/world/src/index.ts | 2 + packages/world/src/slot-identity.test.ts | 29 ++ packages/world/src/slot-identity.ts | 23 ++ 20 files changed, 1248 insertions(+), 123 deletions(-) create mode 100644 .changeset/slot-event-identity-client.md create mode 100644 docs/content/docs/v5/api-reference/workflow-errors/slot-conflict-error.mdx diff --git a/.changeset/slot-event-identity-client.md b/.changeset/slot-event-identity-client.md new file mode 100644 index 0000000000..e6d2c5feee --- /dev/null +++ b/.changeset/slot-event-identity-client.md @@ -0,0 +1,9 @@ +--- +'@workflow/world-vercel': minor +'@workflow/core': minor +'@workflow/errors': minor +'@workflow/world': minor +'workflow': minor +--- + +Event creations on runs that number events by slot now claim their own event id and merge, replay and re-claim when a `SlotConflictError` shows another writer took it first. diff --git a/docs/content/docs/v5/api-reference/workflow-errors/index.mdx b/docs/content/docs/v5/api-reference/workflow-errors/index.mdx index bd4c061ada..540fc5ccef 100644 --- a/docs/content/docs/v5/api-reference/workflow-errors/index.mdx +++ b/docs/content/docs/v5/api-reference/workflow-errors/index.mdx @@ -82,6 +82,9 @@ All errors extend [`WorkflowError`](/docs/api-reference/workflow-errors/workflow Thrown when an event creation is rejected because the client's event-log snapshot is stale. + + Thrown when an event creation is rejected because another writer already took the event's slot. + Thrown when a request is made before the system is ready to process it. diff --git a/docs/content/docs/v5/api-reference/workflow-errors/meta.json b/docs/content/docs/v5/api-reference/workflow-errors/meta.json index a84eddd1ad..b2de769e0e 100644 --- a/docs/content/docs/v5/api-reference/workflow-errors/meta.json +++ b/docs/content/docs/v5/api-reference/workflow-errors/meta.json @@ -15,6 +15,7 @@ "throttle-error", "entity-conflict-error", "precondition-failed-error", + "slot-conflict-error", "run-expired-error", "run-not-supported-error", "too-early-error" diff --git a/docs/content/docs/v5/api-reference/workflow-errors/slot-conflict-error.mdx b/docs/content/docs/v5/api-reference/workflow-errors/slot-conflict-error.mdx new file mode 100644 index 0000000000..96a4f67f17 --- /dev/null +++ b/docs/content/docs/v5/api-reference/workflow-errors/slot-conflict-error.mdx @@ -0,0 +1,76 @@ +--- +title: SlotConflictError +description: Thrown when an event creation is rejected because another writer already took the event's slot. +type: reference +summary: Catch SlotConflictError when a world rejects an event creation whose slot in the run's event log was already taken. +related: + - /docs/api-reference/workflow-errors/workflow-world-error + - /docs/api-reference/workflow-errors/precondition-failed-error +--- + +`SlotConflictError` is thrown by world implementations when an event creation is rejected because the event's slot in the run's event log was already taken by another writer. It corresponds to HTTP 409 Conflict semantics. + +On a run that numbers its events by slot, each event's id encodes its position in the log: the first event is slot 1, the second slot 2, and so on. Whoever writes a slot first owns it, so a rejected write proves the client was replaying against an event log that was missing at least one event. Retrying the same write can never succeed — the client has to merge the events it was missing, replay, and propose whatever slot that replay lands on. + +The rejection carries the missing events inline so that merge usually costs no extra round-trip: + +- `events` — the events recorded after the client's snapshot, in ascending slot order. Empty when the backend could not read them, in which case the client reloads the log itself. +- `cursor` — cursor to continue the delta from. +- `hasMore` — whether events beyond `events` remain to be fetched. + +This is the slot-numbering counterpart to [`PreconditionFailedError`](/docs/api-reference/workflow-errors/precondition-failed-error), which is how the same staleness is reported for runs guarded by an event-log snapshot watermark instead. A run uses one scheme or the other for its whole life, decided when it is created. + + +The Workflow runtime handles this error automatically: it merges the events it was missing, replays, and re-proposes the write at a free slot, ultimately re-enqueueing the run for a fresh replay if it cannot catch up. You will only encounter it when interacting with world storage APIs directly. + + +```typescript lineNumbers +import { SlotConflictError } from "workflow/errors" +declare const world: { events: { create(...args: any[]): Promise } }; // @setup +declare const runId: string; // @setup +declare const event: any; // @setup + +try { + await world.events.create(runId, event); +} catch (error) { + if (SlotConflictError.is(error)) { // [!code highlight] + console.log(`Slot ${error.eventId} taken; ${error.events.length} event(s) missed`); + } +} +``` + +## API Signature + +### Properties + + + +### Static Methods + +#### `SlotConflictError.is(value)` + +Type-safe check for `SlotConflictError` instances. Preferred over `instanceof` because it works across module boundaries and VM contexts. + +```typescript +import { SlotConflictError } from "workflow/errors" +declare const error: unknown; // @setup + +if (SlotConflictError.is(error)) { + // error is typed as SlotConflictError +} +``` diff --git a/packages/core/src/runtime.ts b/packages/core/src/runtime.ts index b8474d2206..86a04e0e9c 100644 --- a/packages/core/src/runtime.ts +++ b/packages/core/src/runtime.ts @@ -48,18 +48,18 @@ import { import { countStepStartedEvents } from './runtime/count-step-started-events.js'; import { appendUniqueEvents, + eventCreateFenceFor, getQueueOverhead, getWorkflowQueueName, handleHealthCheckMessage, isPreconditionGuardEnabled, loadWorkflowRunEvents, - type MutableEventLog, memoizeEncryptionKey, parseHealthCheckPayload, queueMessage, - stateUpdatedAtForCreate, + toMutableEventLog, + withEventCreateFence, withHealthCheck, - withPreconditionRetry, } from './runtime/helpers.js'; import { runScopedKey } from './runtime/idempotency-key.js'; import { @@ -1363,19 +1363,21 @@ export function workflowEntrypoint( }, })); + // One log for the whole loop: `events` is appended to in + // place by the guard's reloads, so a per-iteration + // rescan for the slot high-water mark would be wasted + // work on an array that never changes identity here. + const waitLog = toMutableEventLog(events, eventsCursor); for (const waitEvent of waitsToComplete) { - const waitLog: MutableEventLog = { - events, - cursor: eventsCursor, - }; try { - await withPreconditionRetry( + await withEventCreateFence( runId, waitLog, - (stateUpdatedAt) => + workflowRun.specVersion, + (fence) => world.events.create(runId, waitEvent, { requestId, - stateUpdatedAt, + ...fence, }) ); } catch (err) { @@ -1544,7 +1546,10 @@ export function workflowEntrypoint( }, { requestId, - stateUpdatedAt: stateUpdatedAtForCreate(events), + ...eventCreateFenceFor( + toMutableEventLog(events, eventsCursor), + workflowRun.specVersion + ), } ); } catch (err) { @@ -1612,10 +1617,11 @@ export function workflowEntrypoint( } // V2: handle suspension without queuing steps. - // Each event creation inside handleSuspension carries the - // loaded snapshot's stateUpdatedAt and self-reloads on a - // stale (412) rejection via the shared event log. We - // guard per-create (rather than wrapping the whole call) + // Each event creation inside handleSuspension carries + // the run's concurrency fence — its own event slot, or + // the loaded snapshot's watermark — and self-reloads on + // a rejection (409/412) via the shared event log. We + // fence per-create (rather than wrapping the whole call) // so a retry never re-issues an already-created event. const suspensionStart = Date.now(); // The snapshot refresh above always sets cachedEvents @@ -1629,10 +1635,10 @@ export function workflowEntrypoint( 'Invariant violation: workflow suspended before its event log was loaded' ); } - const suspensionLog: MutableEventLog = { - events: cachedEvents, - cursor: eventsCursor, - }; + const suspensionLog = toMutableEventLog( + cachedEvents, + eventsCursor + ); let suspensionResult: Awaited< ReturnType >; @@ -2114,9 +2120,9 @@ export function workflowEntrypoint( // rejected with 412 — its guarded suspension creates // (retried over the reloaded log, or exhausted into // a queue re-invocation), AND the lazy step_started - // claim of its next inline step, which carries the - // snapshot too (threaded below via - // `stateUpdatedAt`; on rejection the batch is + // claim of its next inline step, which is fenced too + // (threaded below via + // `eventCreateFence`; on rejection the batch is // abandoned and re-invoked for a fresh replay, so a // stale view can never commit a step). Hooks created // by THIS suspension are inside the delta (their @@ -2258,18 +2264,19 @@ export function workflowEntrypoint( turbo, }); - // Precondition-guard snapshot for the inline - // step_started claims: the lazy claim is the first - // durable write of a hot-path step (its step_created - // is deferred), so without a snapshot it would bypass - // the guard entirely and a stale replay could claim — - // and commit — a step scheduled off a view that misses - // an out-of-band event. `stateUpdatedAtForCreate` - // returns undefined when the guard env flag is off, so - // this is a no-op outside guarded deployments; Worlds - // that don't enforce the guard ignore it. - const inlineClaimStateUpdatedAt = - stateUpdatedAtForCreate(cachedEvents ?? []); + // Concurrency fence for the inline step_started claims: + // the lazy claim is the first durable write of a + // hot-path step (its step_created is deferred), so + // without one it would be unguarded and a stale replay + // could claim — and commit — a step scheduled off a view + // that misses an out-of-band event. One log for the + // whole batch so each claim draws its own event slot; + // `eventCreateFenceFor` yields undefined for a run + // fenced neither way, leaving those claims as they were. + const inlineClaimLog = toMutableEventLog( + cachedEvents ?? [], + eventsCursor + ); replayBudget.pause(); let stepResults: Awaited< @@ -2277,6 +2284,15 @@ export function workflowEntrypoint( >[]; const stepExecutionPromises = inlineExecutions.map( (s, stepIndex) => { + // Drawn here — synchronously, in replay order — + // rather than inside `run`: a slot claim is + // positional, so it has to be assigned before these + // executions start racing each other, and the order + // it is assigned in has to be replay-stable. + const eventCreateFence = eventCreateFenceFor( + inlineClaimLog, + workflowRun.specVersion + ); const run = () => executeStep({ world, @@ -2348,7 +2364,7 @@ export function workflowEntrypoint( // see suppressOptimisticStart above. suppressOptimisticStart, runReadyBarrier, - stateUpdatedAt: inlineClaimStateUpdatedAt, + eventCreateFence, ...(stepIndex === 0 && s.lazyStepInput !== undefined && latencyTracking @@ -2738,10 +2754,10 @@ export function workflowEntrypoint( // type identity and custom properties round-trip // through the event log. // - // Precondition-guard asymmetry: unlike `run_completed`, - // this terminal `run_failed` sends no `stateUpdatedAt` - // snapshot, so it is never 412-rejected even if a hook - // landed mid-replay and could have changed the path that + // Fencing asymmetry: unlike `run_completed`, this + // terminal `run_failed` carries no concurrency fence, so + // it is never rejected even if a hook landed mid-replay + // and could have changed the path that // threw. This is intentional and fail-open: a spurious // failure is recoverable (the run can be re-run from the // dashboard), whereas a spurious *completion* commits a diff --git a/packages/core/src/runtime/helpers.test.ts b/packages/core/src/runtime/helpers.test.ts index c6b2927160..15264d404c 100644 --- a/packages/core/src/runtime/helpers.test.ts +++ b/packages/core/src/runtime/helpers.test.ts @@ -1,5 +1,14 @@ -import { PreconditionFailedError, WorkflowWorldError } from '@workflow/errors'; -import type { Event, World } from '@workflow/world'; +import { + PreconditionFailedError, + SlotConflictError, + WorkflowWorldError, +} from '@workflow/errors'; +import { + type Event, + SPEC_VERSION_SLOT_IDENTITY, + slotEventId, + type World, +} from '@workflow/world'; import { ulid } from 'ulid'; import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest'; import { bytesToBase64, deriveRunKeyPair, seal } from '../sealed-box.js'; @@ -11,14 +20,21 @@ import { SerializationFormat, } from '../serialization.js'; import { + eventCreateFenceFor, getWorkflowQueueName, handleHealthCheckMessage, healthCheck, latestEventStateUpdatedAt, loadWorkflowRunEvents, memoizeEncryptionKey, - type MutableEventLog, + mergeLoadedEvents, + PRECONDITION_MAX_RELOAD_RETRIES, + reserveSlot, + stateUpdatedAtForCreate, + toMutableEventLog, + withEventCreateFence, withPreconditionRetry, + withSlotRetry, } from './helpers.js'; // Mock the logger to suppress output during tests @@ -503,6 +519,297 @@ describe('latestEventStateUpdatedAt', () => { }); }); +describe('slot bookkeeping', () => { + const slotEvent = (slot: number) => makeEvent(slotEventId(slot)); + + it('reads maxSlot from the highest slot present, not the last element', () => { + // `appendUniqueEvents` appends without sorting, so a merged log's last + // element is not necessarily its newest event. + const log = toMutableEventLog([slotEvent(3), slotEvent(1)], 'c0'); + expect(log.maxSlot).toBe(3); + expect(log.reserved).toBe(0); + }); + + it('reports maxSlot 0 for an empty or ULID-numbered log', () => { + expect(toMutableEventLog([], null).maxSlot).toBe(0); + expect( + toMutableEventLog([makeUlidEvent(1_700_000_000_000)], null).maxSlot + ).toBe(0); + }); + + it('never lowers maxSlot when an older delta is merged in', () => { + const log = toMutableEventLog([slotEvent(1), slotEvent(3)], 'c0'); + mergeLoadedEvents(log, [slotEvent(2)]); + expect(log.maxSlot).toBe(3); + expect(log.events).toHaveLength(3); + }); + + it('raises maxSlot and drops reservations when a newer delta is merged in', () => { + const log = toMutableEventLog([slotEvent(1)], 'c0'); + reserveSlot(log); + reserveSlot(log); + expect(log.reserved).toBe(2); + + mergeLoadedEvents(log, [slotEvent(2), slotEvent(5)]); + + expect(log.maxSlot).toBe(5); + // The merged events are the authority on which slots are taken, so the + // outstanding reservations (slots 2 and 3) are void. + expect(log.reserved).toBe(0); + expect(reserveSlot(log)).toBe(6); + }); + + it('deduplicates merged events by id', () => { + const log = toMutableEventLog([slotEvent(1)], 'c0'); + mergeLoadedEvents(log, [slotEvent(1), slotEvent(2)]); + expect(log.events.map((e) => e.eventId)).toEqual([ + slotEventId(1), + slotEventId(2), + ]); + }); + + it('hands out contiguous distinct slots for a synchronous burst', () => { + // The suspension flush issues every operation synchronously and awaits + // them together; without contiguous reservation they would all propose the + // same slot and all but one would conflict. + const log = toMutableEventLog([slotEvent(4)], 'c0'); + const burst = Array.from({ length: 20 }, () => reserveSlot(log)); + expect(burst).toEqual(Array.from({ length: 20 }, (_, i) => 5 + i)); + expect(new Set(burst).size).toBe(burst.length); + // Reservations sit past maxSlot rather than moving it: only merged events + // prove a slot is taken. + expect(log.maxSlot).toBe(4); + }); + + it('proposes a padded event id only for a slot-identity run', () => { + const log = toMutableEventLog([], null); + expect(eventCreateFenceFor(log, SPEC_VERSION_SLOT_IDENTITY)).toEqual({ + eventId: slotEventId(1), + maxSlot: 0, + }); + expect(eventCreateFenceFor(log, SPEC_VERSION_SLOT_IDENTITY)).toEqual({ + eventId: slotEventId(2), + maxSlot: 0, + }); + }); + + it('proposes no event id for a ULID-numbered run', () => { + // A run whose ids the backend mints must not burn slots either. + const log = toMutableEventLog([], null); + const fence = eventCreateFenceFor(log, SPEC_VERSION_SLOT_IDENTITY - 1); + expect(fence?.eventId).toBeUndefined(); + expect(log.reserved).toBe(0); + }); +}); + +describe('stateUpdatedAtForCreate', () => { + it('sends no watermark for a slot-numbered run even with the guard on', () => { + // The event id is the fence for these runs. Inference would be worse than + // useless here: a padded slot body is valid Crockford base32, so decoding + // it yields epoch 0 rather than failing, and the client would claim a + // snapshot older than every event in the log. + const events = [makeEvent(slotEventId(1))]; + expect( + stateUpdatedAtForCreate(events, SPEC_VERSION_SLOT_IDENTITY) + ).toBeUndefined(); + }); + + it('sends the snapshot watermark for a ULID-numbered run', () => { + const time = 1_700_000_000_000; + expect( + stateUpdatedAtForCreate( + [makeUlidEvent(time)], + SPEC_VERSION_SLOT_IDENTITY - 1 + ) + ).toBe(time); + }); +}); + +describe('withSlotRetry', () => { + const slotEvent = (slot: number) => makeEvent(slotEventId(slot)); + + beforeEach(() => { + eventsListMock.mockReset(); + }); + + it('claims the next free slot and passes the observed maxSlot alongside it', async () => { + const log = toMutableEventLog([slotEvent(1), slotEvent(2)], 'c0'); + const op = vi.fn(async () => 'ok'); + + await expect(withSlotRetry('wrun_test', log, op)).resolves.toBe('ok'); + expect(op).toHaveBeenCalledWith({ eventId: slotEventId(3), maxSlot: 2 }); + expect(eventsListMock).not.toHaveBeenCalled(); + }); + + it('merges the conflict delta and reclaims past it, without reloading', async () => { + // The inline delta is the whole point of the 409 body: the client learns + // which events it was missing without a follow-up round-trip. + const log = toMutableEventLog([slotEvent(1)], 'c0'); + const claimed: string[] = []; + const op = vi.fn(async ({ eventId }: { eventId?: string }) => { + claimed.push(eventId as string); + if (claimed.length === 1) { + throw new SlotConflictError('taken', { + eventId: eventId as string, + events: [slotEvent(2), slotEvent(3)], + cursor: 'c1', + }); + } + return 'done'; + }); + + await expect(withSlotRetry('wrun_test', log, op)).resolves.toBe('done'); + expect(claimed).toEqual([slotEventId(2), slotEventId(4)]); + expect(log.events).toHaveLength(3); + expect(log.cursor).toBe('c1'); + expect(eventsListMock).not.toHaveBeenCalled(); + }); + + it('tops the delta up from the backend when it was truncated', async () => { + const log = toMutableEventLog([slotEvent(1)], 'c0'); + eventsListMock.mockResolvedValueOnce({ + data: [slotEvent(3)], + cursor: 'c2', + hasMore: false, + }); + let attempts = 0; + const op = vi.fn(async () => { + attempts++; + if (attempts === 1) { + throw new SlotConflictError('taken', { + eventId: slotEventId(2), + events: [slotEvent(2)], + cursor: 'c1', + hasMore: true, + }); + } + return 'done'; + }); + + await expect(withSlotRetry('wrun_test', log, op)).resolves.toBe('done'); + expect(eventsListMock).toHaveBeenCalledTimes(1); + expect(log.maxSlot).toBe(3); + expect(op).toHaveBeenLastCalledWith({ + eventId: slotEventId(4), + maxSlot: 3, + }); + }); + + it('falls back to a full incremental load when the rejection carried no delta', async () => { + const log = toMutableEventLog([slotEvent(1)], 'c0'); + eventsListMock.mockResolvedValueOnce({ + data: [slotEvent(2)], + cursor: 'c1', + hasMore: false, + }); + let attempts = 0; + const op = vi.fn(async () => { + attempts++; + if (attempts === 1) { + throw new SlotConflictError('taken', { eventId: slotEventId(2) }); + } + return 'done'; + }); + + await expect(withSlotRetry('wrun_test', log, op)).resolves.toBe('done'); + expect(eventsListMock).toHaveBeenCalledTimes(1); + expect(op).toHaveBeenLastCalledWith({ + eventId: slotEventId(3), + maxSlot: 2, + }); + }); + + it('rethrows the conflict once the reclaim budget is spent', async () => { + // Escaping to a fresh replay is the correct fallback, not a failure mode: + // the merged events can change what the workflow body decides, and only a + // replay from the top can act on that. + const log = toMutableEventLog([slotEvent(1)], 'c0'); + eventsListMock.mockResolvedValue({ + data: [], + cursor: 'c1', + hasMore: false, + }); + const op = vi.fn(async ({ eventId }: { eventId?: string }) => { + throw new SlotConflictError('taken', { eventId: eventId as string }); + }); + + await expect(withSlotRetry('wrun_test', log, op)).rejects.toBeInstanceOf( + SlotConflictError + ); + expect(op).toHaveBeenCalledTimes(PRECONDITION_MAX_RELOAD_RETRIES + 1); + expect(eventsListMock).toHaveBeenCalledTimes( + PRECONDITION_MAX_RELOAD_RETRIES + ); + }); + + it('rethrows a non-conflict error immediately, without merging', async () => { + const log = toMutableEventLog([slotEvent(1)], 'c0'); + const op = vi.fn(async () => { + throw new PreconditionFailedError('stale'); + }); + + await expect(withSlotRetry('wrun_test', log, op)).rejects.toBeInstanceOf( + PreconditionFailedError + ); + expect(op).toHaveBeenCalledTimes(1); + expect(eventsListMock).not.toHaveBeenCalled(); + }); +}); + +describe('withEventCreateFence', () => { + beforeEach(() => { + eventsListMock.mockReset(); + }); + + it('fences a slot-numbered run by event id and retries its 409s', async () => { + const log = toMutableEventLog([makeEvent(slotEventId(1))], 'c0'); + let attempts = 0; + const op = vi.fn(async ({ eventId }: { eventId?: string }) => { + attempts++; + if (attempts === 1) { + throw new SlotConflictError('taken', { + eventId: eventId as string, + events: [makeEvent(slotEventId(2))], + cursor: 'c1', + }); + } + return 'done'; + }); + + await expect( + withEventCreateFence('wrun_test', log, SPEC_VERSION_SLOT_IDENTITY, op) + ).resolves.toBe('done'); + expect(op).toHaveBeenLastCalledWith({ + eventId: slotEventId(3), + maxSlot: 2, + }); + }); + + it('fences a ULID-numbered run by watermark and retries its 412s', async () => { + const time = 1_700_000_000_000; + const log = toMutableEventLog([makeUlidEvent(time)], 'c0'); + eventsListMock.mockResolvedValueOnce({ + data: [makeUlidEvent(time + 1000)], + cursor: 'c1', + hasMore: false, + }); + let attempts = 0; + const op = vi.fn(async () => { + attempts++; + if (attempts === 1) { + throw new PreconditionFailedError('stale'); + } + return 'done'; + }); + + await expect( + withEventCreateFence('wrun_test', log, SPEC_VERSION_SLOT_IDENTITY - 1, op) + ).resolves.toBe('done'); + expect(op).toHaveBeenLastCalledWith({ stateUpdatedAt: time + 1000 }); + expect(eventsListMock).toHaveBeenCalledTimes(1); + }); +}); + describe('withPreconditionRetry', () => { let originalGuard: string | undefined; @@ -522,10 +829,7 @@ describe('withPreconditionRetry', () => { it('passes no snapshot to op when the guard is explicitly disabled', async () => { process.env.WORKFLOW_PRECONDITION_GUARD = '0'; - const log: MutableEventLog = { - events: [makeUlidEvent(1_700_000_000_000)], - cursor: 'c0', - }; + const log = toMutableEventLog([makeUlidEvent(1_700_000_000_000)], 'c0'); const op = vi.fn(async (stateUpdatedAt?: number) => { expect(stateUpdatedAt).toBeUndefined(); return 'ok'; @@ -541,10 +845,7 @@ describe('withPreconditionRetry', () => { it('sends a snapshot by default when the guard variable is unset (on by default)', async () => { delete process.env.WORKFLOW_PRECONDITION_GUARD; const time = 1_700_000_000_000; - const log: MutableEventLog = { - events: [makeUlidEvent(time)], - cursor: 'c0', - }; + const log = toMutableEventLog([makeUlidEvent(time)], 'c0'); const op = vi.fn(async (stateUpdatedAt?: number) => { expect(stateUpdatedAt).toBe(time); return 'ok'; @@ -558,10 +859,7 @@ describe('withPreconditionRetry', () => { it('passes the latest snapshot time to op and returns its result without reloading', async () => { const time = 1_700_000_000_000; - const log: MutableEventLog = { - events: [makeUlidEvent(time)], - cursor: 'c0', - }; + const log = toMutableEventLog([makeUlidEvent(time)], 'c0'); const op = vi.fn(async (stateUpdatedAt?: number) => { expect(stateUpdatedAt).toBe(time); return 'ok'; @@ -575,10 +873,7 @@ describe('withPreconditionRetry', () => { }); it('reloads the event log and retries on a stale (412) rejection, then succeeds', async () => { - const log: MutableEventLog = { - events: [makeUlidEvent(1_700_000_000_000)], - cursor: 'c0', - }; + const log = toMutableEventLog([makeUlidEvent(1_700_000_000_000)], 'c0'); // Each reload returns one newer event and advances the cursor. eventsListMock.mockResolvedValueOnce({ data: [makeUlidEvent(1_700_000_001_000)], @@ -610,10 +905,7 @@ describe('withPreconditionRetry', () => { }); it('rethrows the precondition error after exhausting reload retries', async () => { - const log: MutableEventLog = { - events: [makeUlidEvent(1_700_000_000_000)], - cursor: 'c0', - }; + const log = toMutableEventLog([makeUlidEvent(1_700_000_000_000)], 'c0'); eventsListMock.mockResolvedValue({ data: [], cursor: 'c1', @@ -633,10 +925,7 @@ describe('withPreconditionRetry', () => { }); it('rethrows non-precondition errors immediately without reloading', async () => { - const log: MutableEventLog = { - events: [makeUlidEvent(1_700_000_000_000)], - cursor: 'c0', - }; + const log = toMutableEventLog([makeUlidEvent(1_700_000_000_000)], 'c0'); const op = vi.fn(async () => { throw new Error('boom'); }); diff --git a/packages/core/src/runtime/helpers.ts b/packages/core/src/runtime/helpers.ts index 8664548658..0dda520104 100644 --- a/packages/core/src/runtime/helpers.ts +++ b/packages/core/src/runtime/helpers.ts @@ -1,6 +1,7 @@ import { PreconditionFailedError, RUN_ERROR_CODES, + SlotConflictError, WorkflowWorldError, } from '@workflow/errors'; import type { @@ -13,10 +14,13 @@ import type { import { getQueueTopicPrefix, HealthCheckPayloadSchema, + maxSlotOf, resolveQueueNamespace, SPEC_VERSION_CURRENT, SPEC_VERSION_LEGACY, + slotEventId, ulidToDate, + usesSlotIdentity, } from '@workflow/world'; import { monotonicFactory } from 'ulid'; import { runtimeLogger } from '../logger.js'; @@ -650,6 +654,55 @@ export const PRECONDITION_MAX_RELOAD_RETRIES = 2; export interface MutableEventLog { events: Event[]; cursor: string | null; + /** + * Highest slot present in `events`, or 0 for a log that is empty or + * ULID-numbered. Maintained by `mergeLoadedEvents` from the events merged in, + * never from the array's last element: events are appended without sorting, + * so after a merge the last element need not be the newest. + */ + maxSlot: number; + /** + * Slots handed out by `reserveSlot` past `maxSlot` whose events have not been + * merged back yet. Reset whenever the log is merged into, because the merged + * events are the authority on which slots are taken. + */ + reserved: number; +} + +/** A `MutableEventLog` over a freshly loaded snapshot. */ +export function toMutableEventLog( + events: Event[], + cursor: string | null +): MutableEventLog { + return { events, cursor, maxSlot: maxSlotOf(events), reserved: 0 }; +} + +/** + * Merges loaded events into `log` in place, keeping `maxSlot` current and + * dropping outstanding reservations (the merged events supersede them). + */ +export function mergeLoadedEvents( + log: MutableEventLog, + events: readonly Event[] +): void { + appendUniqueEvents(log.events, events); + log.maxSlot = Math.max(log.maxSlot, maxSlotOf(events)); + log.reserved = 0; +} + +/** + * Claims the next free slot in `log`, synchronously at the moment an event is + * issued. + * + * Reservations are contiguous rather than all-`maxSlot + 1` because a + * suspension flushes its operations concurrently: without them every operation + * in the flush would propose the same slot and all but one would conflict, on + * every single flush. Operations are built in deterministic replay order, so + * the slot each one draws is replay-stable too. + */ +export function reserveSlot(log: MutableEventLog): number { + log.reserved += 1; + return log.maxSlot + log.reserved; } /** @@ -703,8 +756,23 @@ export function latestEventStateUpdatedAt(events: Event[]): number | undefined { * The `stateUpdatedAt` to attach to a replay-context event creation: * the loaded snapshot's ULID time when the precondition guard is enabled, * `undefined` (no guard, backend behaves as before) otherwise. + * + * Always `undefined` for a run that numbers its events by slot, where the event + * id is itself the concurrency fence and the watermark is dead weight. The mode + * is passed in explicitly rather than inferred from the event ids, because + * inference would silently produce a *wrong* value instead of none: a padded + * slot body is valid Crockford base32, so `latestEventStateUpdatedAt` decodes it + * to epoch 0 rather than failing its fail-open check, and the client would send + * `stateUpdatedAt: 0` — which a backend still running the watermark guard reads + * as a snapshot older than every event. */ -export function stateUpdatedAtForCreate(events: Event[]): number | undefined { +export function stateUpdatedAtForCreate( + events: Event[], + specVersion?: number +): number | undefined { + if (usesSlotIdentity(specVersion)) { + return undefined; + } return isPreconditionGuardEnabled() ? latestEventStateUpdatedAt(events) : undefined; @@ -749,7 +817,7 @@ export async function withPreconditionRetry( runId, log.cursor ?? undefined ); - appendUniqueEvents(log.events, loaded.events); + mergeLoadedEvents(log, loaded.events); // When several creates share one `log` (e.g. hook creations under // `Promise.all` in `handleSuspension`), concurrent 412s can reload // concurrently. The event merge above is safe — `appendUniqueEvents` @@ -762,6 +830,138 @@ export async function withPreconditionRetry( } } +/** + * The concurrency fence a replay-context event creation carries. Exactly one of + * the two schemes is ever populated: `stateUpdatedAt` for a run guarded by the + * event-log watermark, `eventId`/`maxSlot` for a run that numbers its events by + * slot. + */ +export interface EventCreateFence { + stateUpdatedAt?: number; + eventId?: string; + maxSlot?: number; +} + +/** + * The fence for a create that is deliberately **not** retried in place, because + * a rejection means the committed decision itself is stale and only a fresh + * replay can revise it (`run_completed`, an inline `step_started` claim). + * + * Claims a slot off `log` for a slot-numbered run — which counts as a + * reservation, so a caller that fences several creates from one log gets a + * distinct slot per create. `undefined` when the run is fenced neither way, + * leaving the create exactly as unfenced as it was before either mechanism + * existed. + */ +export function eventCreateFenceFor( + log: MutableEventLog, + specVersion: number | undefined +): EventCreateFence | undefined { + if (usesSlotIdentity(specVersion)) { + return { eventId: slotEventId(reserveSlot(log)), maxSlot: log.maxSlot }; + } + const stateUpdatedAt = stateUpdatedAtForCreate(log.events, specVersion); + return stateUpdatedAt !== undefined ? { stateUpdatedAt } : undefined; +} + +/** + * Runs a replay-context event creation that claims its own event slot. + * + * The claim is the fence: the backend inserts the proposed `eventId` + * conditionally, so a `SlotConflictError` (409) proves another writer got there + * first and that this replay therefore ran against an event log missing at least + * one event. Re-sending the same write is pointless — it would lose the same + * slot again — so each attempt merges the events it was missing (inline off the + * rejection, topped up from the backend when the delta was truncated) and claims + * a fresh slot past them. + * + * Bounded by `PRECONDITION_MAX_RELOAD_RETRIES`, after which the error + * propagates and the run is re-invoked from the queue for a fresh replay. That + * fallback is not merely a giving-up path: merged events can change what the + * workflow body decides, and only a replay from the top can act on them. + */ +export async function withSlotRetry( + runId: string, + log: MutableEventLog, + op: (fence: EventCreateFence) => Promise +): Promise { + for (let attempt = 0; ; attempt++) { + // Claimed per attempt, not once up front: a merged delta moves the log's + // high-water mark, so the previous claim is stale by definition. + const maxSlot = log.maxSlot; + const eventId = slotEventId(reserveSlot(log)); + try { + return await op({ eventId, maxSlot }); + } catch (error) { + if ( + !SlotConflictError.is(error) || + attempt >= PRECONDITION_MAX_RELOAD_RETRIES + ) { + throw error; + } + runtimeLogger.info( + 'Event creation lost its slot; merging missed events and reclaiming', + { + workflowRunId: runId, + eventId, + attempt: attempt + 1, + maxRetries: PRECONDITION_MAX_RELOAD_RETRIES, + } + ); + await mergeSlotConflictDelta(runId, log, error); + } + } +} + +/** + * Merges the event-log delta a slot conflict carries into `log`. + * + * The inline delta is an optimization, not a contract: a backend that could not + * read it sends none, and one that paginated it sets `hasMore`. Either way the + * fallback is the same full incremental load the runtime would otherwise have + * done, so the merged log is authoritative in every case. + */ +async function mergeSlotConflictDelta( + runId: string, + log: MutableEventLog, + conflict: SlotConflictError +): Promise { + const inline = conflict.events as Event[]; + if (inline.length > 0) { + mergeLoadedEvents(log, inline); + log.cursor = conflict.cursor ?? log.cursor; + } + if (inline.length === 0 || conflict.hasMore) { + const loaded = await loadWorkflowRunEvents(runId, log.cursor ?? undefined); + mergeLoadedEvents(log, loaded.events); + log.cursor = loaded.cursor ?? log.cursor; + } +} + +/** + * Runs a replay-context event creation under whichever concurrency fence the + * run uses: its event slot when it numbers events by slot, the event-log + * watermark otherwise. + * + * The two retry loops stay separate rather than being folded together. They + * differ in what a rejection proves and in what the client does about it, and + * both are live at once while runs on the older numbering drain — keeping them + * apart is what makes a rollout's 409s and 412s separately countable. + */ +export function withEventCreateFence( + runId: string, + log: MutableEventLog, + specVersion: number | undefined, + op: (fence: EventCreateFence) => Promise +): Promise { + if (usesSlotIdentity(specVersion)) { + return withSlotRetry(runId, log, op); + } + return withPreconditionRetry(runId, log, (stateUpdatedAt) => + op({ stateUpdatedAt }) + ); +} + /** * CORS headers for health check responses. * Allows the observability UI to check endpoint health from a different origin. diff --git a/packages/core/src/runtime/step-executor.ts b/packages/core/src/runtime/step-executor.ts index 566f7d23d3..e1823cfb24 100644 --- a/packages/core/src/runtime/step-executor.ts +++ b/packages/core/src/runtime/step-executor.ts @@ -18,9 +18,9 @@ import { SPEC_VERSION_CURRENT, SPEC_VERSION_SUPPORTS_COMPRESSION, } from '@workflow/world'; -import type { PayloadKey } from '../serialization/encryption.js'; import { runtimeLogger, stepLogger } from '../logger.js'; import { getStepFunction } from '../private.js'; +import type { PayloadKey } from '../serialization/encryption.js'; import { cancelAbortReaders, dehydrateStepError, @@ -43,7 +43,7 @@ import { isOptimisticInlineStartExplicitlyDisabled, } from './constants.js'; import { getPortLazy } from './get-port-lazy.js'; -import { memoizeEncryptionKey } from './helpers.js'; +import { type EventCreateFence, memoizeEncryptionKey } from './helpers.js'; import { computeStepLatencyEventData, type StepLatencyEventData, @@ -129,20 +129,23 @@ export interface StepExecutorParams { */ inlineDeltaSinceCursor?: string; /** - * Precondition-guard snapshot (epoch ms of the latest event the caller's - * replay loaded) to attach to this step's `step_started` claim. On the lazy - * inline path the claim is the step's FIRST durable write (its - * `step_created` is deferred), so without this the claim would bypass the - * optimistic-concurrency guard entirely: a replay working from a stale view - * could claim — and then commit — a step scheduled without observing an - * out-of-band event. A guard-enforcing World rejects a stale claim with - * `PreconditionFailedError` (412); executeStep does NOT translate that - * rejection (re-claiming in place would still commit the stale schedule), - * so it propagates for the caller to abandon the batch and force a fresh - * replay. Undefined when the guard is disabled or the caller has no - * snapshot; Worlds that don't enforce the guard ignore it. + * Concurrency fence to attach to this step's `step_started` claim: the event + * slot the claim occupies, or the caller's replay snapshot (`stateUpdatedAt`, + * epoch ms of the latest event it loaded) for a run on the older numbering. + * + * On the lazy inline path the claim is the step's FIRST durable write (its + * `step_created` is deferred), so without a fence it would be unguarded + * entirely: a replay working from a stale view could claim — and then commit — + * a step scheduled without observing an out-of-band event. A fencing World + * rejects such a claim with `SlotConflictError` (409) or + * `PreconditionFailedError` (412); executeStep does NOT translate either + * rejection (re-claiming in place would still commit the stale schedule), so + * it propagates for the caller to abandon the batch and force a fresh replay. + * + * Undefined when the caller has no snapshot, or when the watermark guard is + * disabled on a run that uses it; Worlds that fence neither way ignore it. */ - stateUpdatedAt?: number; + eventCreateFence?: EventCreateFence; /** * Suppress optimistic inline start for this step regardless of * `WORKFLOW_OPTIMISTIC_INLINE_START` / `forceOptimisticStart`: take the @@ -556,13 +559,11 @@ export async function executeStep( : {}), }, }, - // Guard the claim — see StepExecutorParams.stateUpdatedAt. A stale - // (412) rejection surfaces via reconcileOptimisticStart as a + // Fence the claim — see StepExecutorParams.eventCreateFence. A 409/412 + // rejection surfaces via reconcileOptimisticStart as a // non-translatable error: the body result is discarded and the // rejection propagates to the caller. - params.stateUpdatedAt !== undefined - ? { stateUpdatedAt: params.stateUpdatedAt } - : undefined + params.eventCreateFence ); } ); @@ -619,13 +620,10 @@ export async function executeStep( } : { stepName, ...ownershipStamp }, }, - // Guard the claim — see StepExecutorParams.stateUpdatedAt. A stale - // (412) rejection is intentionally NOT translated by - // startErrorToResult below, so it propagates to the caller for a - // fresh replay. - params.stateUpdatedAt !== undefined - ? { stateUpdatedAt: params.stateUpdatedAt } - : undefined + // Fence the claim — see StepExecutorParams.eventCreateFence. A 409/412 + // rejection is intentionally NOT translated by startErrorToResult + // below, so it propagates to the caller for a fresh replay. + params.eventCreateFence ); if (!startResult.step) { diff --git a/packages/core/src/runtime/suspension-handler.ts b/packages/core/src/runtime/suspension-handler.ts index 921986d7bd..81e1760049 100644 --- a/packages/core/src/runtime/suspension-handler.ts +++ b/packages/core/src/runtime/suspension-handler.ts @@ -30,7 +30,7 @@ import { dehydrateStepArguments } from '../serialization.js'; import * as Attribute from '../telemetry/semantic-conventions.js'; import { getAbortStreamIdFromToken } from '../util.js'; import { getMaxInlineSteps } from './constants.js'; -import { type MutableEventLog, withPreconditionRetry } from './helpers.js'; +import { type MutableEventLog, withEventCreateFence } from './helpers.js'; export interface SuspensionHandlerParams { suspension: WorkflowSuspension; @@ -39,10 +39,11 @@ export interface SuspensionHandlerParams { span?: Span; requestId?: string; /** - * The runtime's loaded event log. Each event creation is sent with this - * snapshot's `stateUpdatedAt` and, if the backend rejects it as stale (412), - * the log is reloaded in place and the create is retried — see - * `withPreconditionRetry`. Guarding per-create (rather than the whole + * The runtime's loaded event log. Each event creation carries a fence derived + * from this snapshot — its own event slot, or the snapshot's `stateUpdatedAt` + * for a run on the older numbering — and if the backend rejects the write, the + * log is reloaded in place and the create is retried; see + * `withEventCreateFence`. Fencing per-create (rather than the whole * suspension) ensures a retry never re-issues an already-created event. */ eventLog?: MutableEventLog; @@ -226,11 +227,12 @@ export async function handleSuspension({ } }; - // Create an event with the optimistic-concurrency guard when the caller - // supplied a loaded event log; otherwise create it directly (callers without - // a replay snapshot, e.g. tests). The guard reloads + retries on a stale - // (412) rejection, keeping `eventLog` current in place. All suspension events - // are non-run_created events on this run's `runId`. + // Create an event under the run's concurrency fence when the caller supplied + // a loaded event log; otherwise create it directly (callers without a replay + // snapshot, e.g. tests). The fence reloads + retries on a rejection, keeping + // `eventLog` current in place. Fencing per-create rather than per-suspension + // is what makes a retry safe: it never re-issues an already-created event. + // All suspension events are non-run_created events on this run's `runId`. const createGuarded = ( data: CreateEventRequest, params?: CreateEventParams @@ -238,8 +240,8 @@ export async function handleSuspension({ if (!eventLog) { return world.events.create(runId, data, params); } - return withPreconditionRetry(runId, eventLog, (stateUpdatedAt) => - world.events.create(runId, data, { ...params, stateUpdatedAt }) + return withEventCreateFence(runId, eventLog, run.specVersion, (fence) => + world.events.create(runId, data, { ...params, ...fence }) ); }; // Separate queue items by type diff --git a/packages/errors/src/index.ts b/packages/errors/src/index.ts index 3e0700053f..96e4399c46 100644 --- a/packages/errors/src/index.ts +++ b/packages/errors/src/index.ts @@ -809,6 +809,59 @@ export class PreconditionFailedError extends WorkflowWorldError { } } +/** + * Thrown when the backend rejects an event creation because the event slot the + * client named was already taken by another writer (HTTP 409). + * + * On a run that numbers its events by slot, whoever writes a slot first owns + * it, and the loser has by definition been replaying against an event log + * missing at least one event. Retrying the same write can therefore never + * succeed: the client has to merge the events it was missing, replay, and + * propose whatever slot that replay lands on. The rejection carries those + * events inline so the common case costs no extra round-trip. + * + * Distinct from `PreconditionFailedError` (412), which is the equivalent + * rejection for a run guarded by the `stateUpdatedAt` watermark instead. Both + * mechanisms are live at once while runs on the older numbering drain. + * + * The workflow runtime handles this automatically. Users interacting with world + * storage backends directly may encounter it. + * + * @property eventId - The slot-numbered event id that was already taken. + * @property events - The events recorded after the client's snapshot, in + * ascending slot order. Empty when the backend could not read them, in which + * case the client reloads the log itself. + * @property cursor - Cursor to continue the delta from, or `null`. + * @property hasMore - Whether events beyond `events` remain to be fetched. + */ +export class SlotConflictError extends WorkflowWorldError { + readonly eventId: string; + readonly events: unknown[]; + readonly cursor: string | null; + readonly hasMore: boolean; + + constructor( + message: string, + options: { + eventId: string; + events?: unknown[]; + cursor?: string | null; + hasMore?: boolean; + } + ) { + super(message, { status: 409 }); + this.name = 'SlotConflictError'; + this.eventId = options.eventId; + this.events = options.events ?? []; + this.cursor = options.cursor ?? null; + this.hasMore = options.hasMore ?? false; + } + + static is(value: unknown): value is SlotConflictError { + return isError(value) && value.name === 'SlotConflictError'; + } +} + /** * Thrown when awaiting `run.returnValue` on a workflow run that was cancelled. * diff --git a/packages/workflow/src/internal/errors.ts b/packages/workflow/src/internal/errors.ts index 4490e7ed4f..9e0cdb3774 100644 --- a/packages/workflow/src/internal/errors.ts +++ b/packages/workflow/src/internal/errors.ts @@ -5,6 +5,7 @@ export { PreconditionFailedError, RunExpiredError, RunNotSupportedError, + SlotConflictError, StepNotRegisteredError, ThrottleError, TooEarlyError, diff --git a/packages/world-vercel/src/event-retry.test.ts b/packages/world-vercel/src/event-retry.test.ts index d514576c7b..c0b7f65491 100644 --- a/packages/world-vercel/src/event-retry.test.ts +++ b/packages/world-vercel/src/event-retry.test.ts @@ -1,6 +1,7 @@ import { EntityConflictError, RunExpiredError, + SlotConflictError, ThrottleError, TooEarlyError, WorkflowWorldError, @@ -94,6 +95,16 @@ describe('isRetryableEventPostError', () => { expect(isRetryableEventPostError(new ThrottleError('429'))).toBe(false); }); + it('does not retry a lost event slot', () => { + // Re-issuing the same write is guaranteed to lose the slot again; only a + // merge and a replay can produce a write that lands. + expect( + isRetryableEventPostError( + new SlotConflictError('taken', { eventId: 'evnt_x' }) + ) + ).toBe(false); + }); + it('retries a body-parse failure (write may have landed)', () => { expect( isRetryableEventPostError( diff --git a/packages/world-vercel/src/event-retry.ts b/packages/world-vercel/src/event-retry.ts index fc0d3bd87f..25985a6c5c 100644 --- a/packages/world-vercel/src/event-retry.ts +++ b/packages/world-vercel/src/event-retry.ts @@ -40,6 +40,7 @@ import { EntityConflictError, RunExpiredError, + SlotConflictError, ThrottleError, TooEarlyError, WorkflowWorldError, @@ -203,9 +204,13 @@ function collectErrorMarkers(err: unknown, depth = 0): string[] { export function isRetryableEventPostError(err: unknown): boolean { // Definitive, server-considered outcomes — never retried in-process. // (425/429 are intentionally left to the runtime's retry-after handling.) + // A slot conflict is doubly definitive: the write lost a race for its event + // id, so re-issuing it unchanged is guaranteed to lose again. Only a merge and + // a replay can produce a write that can land. if ( EntityConflictError.is(err) || RunExpiredError.is(err) || + SlotConflictError.is(err) || TooEarlyError.is(err) || ThrottleError.is(err) ) { diff --git a/packages/world-vercel/src/events-v4.test.ts b/packages/world-vercel/src/events-v4.test.ts index a22e3fc12c..2139d8c3eb 100644 --- a/packages/world-vercel/src/events-v4.test.ts +++ b/packages/world-vercel/src/events-v4.test.ts @@ -1,20 +1,26 @@ import { EntityConflictError, RunExpiredError, + SlotConflictError, ThrottleError, TooEarlyError, WorkflowWorldError, } from '@workflow/errors'; +import { + slotEventId, + slotIdBody, + SPEC_VERSION_SLOT_IDENTITY, +} from '@workflow/world'; import { decode, encode } from 'cbor-x'; import { MockAgent } from 'undici'; import { afterEach, describe, expect, it, vi } from 'vitest'; +import { splitEventDataForV4 } from './events.js'; import { createWorkflowRunEventV4, getEventV4, getWorkflowRunEventsV4, throwForErrorResponse, } from './events-v4.js'; -import { splitEventDataForV4 } from './events.js'; import { encodeFrame, V4_FRAME_CONTENT_TYPE } from './frames.js'; import { WORKFLOW_SERVER_URL_OVERRIDE } from './utils.js'; @@ -27,7 +33,7 @@ import { WORKFLOW_SERVER_URL_OVERRIDE } from './utils.js'; describe('throwForErrorResponse', () => { const call = ( status: number, - body = '{"message":"boom"}', + body: string | Uint8Array = '{"message":"boom"}', headers: Record = {} ) => throwForErrorResponse(status, headers, body, 'createEvent', 'http://x'); @@ -107,6 +113,109 @@ describe('throwForErrorResponse', () => { /createEvent failed: HTTP 500 plain text oops/ ); }); + + /** + * The slot-conflict 409 is the only v4 error body that arrives as CBOR: it + * carries the event-log delta the client replays from, whose payloads are + * byte strings that JSON cannot represent. Decoding it as JSON would lose the + * delta silently and mis-type the error as an entity conflict, which the + * runtime reads as "my write already landed". + */ + describe('slot conflict', () => { + const PAYLOAD = new Uint8Array([1, 2, 3]); + const conflictBody = ( + overrides: Record = {} + ): Uint8Array => + new Uint8Array( + encode({ + success: false, + error: 'slot-conflict', + message: "Event slot 'evnt_…003' is already taken", + details: { eventId: 'evnt_from_details' }, + events: [{ eventId: 'evnt_x', eventData: { output: PAYLOAD } }], + cursor: 'eid:evnt_x', + hasMore: false, + ...overrides, + }) + ); + + it('decodes a CBOR body into SlotConflictError with the delta intact', () => { + try { + call(409, conflictBody(), { + 'content-type': 'application/cbor', + 'x-wf-event-id': 'evnt_from_header', + }); + expect.unreachable(); + } catch (err) { + expect(SlotConflictError.is(err)).toBe(true); + const conflict = err as SlotConflictError; + expect(conflict.eventId).toBe('evnt_from_header'); + expect(conflict.cursor).toBe('eid:evnt_x'); + expect(conflict.hasMore).toBe(false); + // Binary payloads survive: this is what a JSON error path destroys. + expect(conflict.events).toHaveLength(1); + expect( + (conflict.events[0] as { eventData: { output: Uint8Array } }) + .eventData.output + ).toEqual(PAYLOAD); + // Not the 409 → EntityConflictError mapping, which the runtime reads as + // "the write I am retrying already landed". + expect(EntityConflictError.is(err)).toBe(false); + } + }); + + it('falls back to the eventId in details when the header is absent', () => { + try { + call(409, conflictBody(), { 'content-type': 'application/cbor' }); + expect.unreachable(); + } catch (err) { + expect((err as SlotConflictError).eventId).toBe('evnt_from_details'); + } + }); + + it('reports an empty delta when the backend could not read one', () => { + try { + call(409, conflictBody({ events: undefined, cursor: null }), { + 'content-type': 'application/cbor', + }); + expect.unreachable(); + } catch (err) { + const conflict = err as SlotConflictError; + expect(conflict.events).toEqual([]); + expect(conflict.cursor).toBeNull(); + } + }); + + it('reads a JSON-encoded slot conflict too', () => { + // Nothing in the protocol forbids a JSON encoding of the same body; only + // the delta's binary payloads require CBOR. + try { + call( + 409, + JSON.stringify({ + error: 'slot-conflict', + message: 'taken', + events: [], + cursor: 'eid:evnt_y', + hasMore: true, + }), + { 'x-wf-event-id': 'evnt_j' } + ); + expect.unreachable(); + } catch (err) { + expect(SlotConflictError.is(err)).toBe(true); + expect((err as SlotConflictError).hasMore).toBe(true); + } + }); + + it('leaves an ordinary 409 as EntityConflictError', () => { + // Entity materialization conflicts share the status and are how the + // runtime recognizes a duplicate write. + expect(() => + call(409, JSON.stringify({ error: 'conflict', message: 'exists' })) + ).toThrowError(EntityConflictError); + }); + }); }); /** @@ -612,6 +721,117 @@ describe('createWorkflowRunEventV4 over HTTP', () => { expect('stateUpdatedAt' in (capturedMeta ?? {})).toBe(false); agent.assertNoPendingInterceptors(); }); + + it('sends the claimed eventId and maxSlot in the frame meta', async () => { + // A slot-numbered run names its own event ids, so the id has to reach the + // wire: the backend reads it from the frame meta and inserts it + // conditionally. Dropped, the backend mints a ULID instead and the run + // silently loses the density its completeness check depends on. + const origin = + WORKFLOW_SERVER_URL_OVERRIDE || 'https://vercel-workflow.com'; + const agent = new MockAgent(); + agent.disableNetConnect(); + + let capturedMeta: Record | undefined; + agent + .get(origin) + .intercept({ + path: '/api/v4/runs/wrun_1/events/wait_created', + method: 'POST', + }) + .reply( + 200, + (opts: { body?: unknown }) => { + const bytes = new Uint8Array(opts.body as ArrayBufferLike); + const metaLen = new DataView( + bytes.buffer, + bytes.byteOffset, + bytes.byteLength + ).getUint32(0, false); + capturedMeta = decode(bytes.subarray(4, 4 + metaLen)) as Record< + string, + unknown + >; + return encode({ wait: { waitId: 'wait_1' } }); + }, + { + headers: { + 'x-wf-event-id': 'evnt_1', + 'x-wf-run-id': 'wrun_1', + 'x-wf-created-at': '2026-06-10T00:00:00.000Z', + }, + } + ); + + const eventId = slotEventId(4); + await createWorkflowRunEventV4( + { + runId: 'wrun_1', + eventType: 'wait_created', + specVersion: SPEC_VERSION_SLOT_IDENTITY, + correlationId: `wait_${slotIdBody(1)}`, + eventId, + maxSlot: 3, + }, + { token: 'test-token', dispatcher: agent } + ); + + expect(capturedMeta?.eventId).toBe(eventId); + expect(capturedMeta?.maxSlot).toBe(3); + agent.assertNoPendingInterceptors(); + }); + + it('omits eventId and maxSlot from the frame meta for a ULID-numbered run', async () => { + const origin = + WORKFLOW_SERVER_URL_OVERRIDE || 'https://vercel-workflow.com'; + const agent = new MockAgent(); + agent.disableNetConnect(); + + let capturedMeta: Record | undefined; + agent + .get(origin) + .intercept({ + path: '/api/v4/runs/wrun_1/events/wait_created', + method: 'POST', + }) + .reply( + 200, + (opts: { body?: unknown }) => { + const bytes = new Uint8Array(opts.body as ArrayBufferLike); + const metaLen = new DataView( + bytes.buffer, + bytes.byteOffset, + bytes.byteLength + ).getUint32(0, false); + capturedMeta = decode(bytes.subarray(4, 4 + metaLen)) as Record< + string, + unknown + >; + return encode({ wait: { waitId: 'wait_1' } }); + }, + { + headers: { + 'x-wf-event-id': 'evnt_1', + 'x-wf-run-id': 'wrun_1', + 'x-wf-created-at': '2026-06-10T00:00:00.000Z', + }, + } + ); + + await createWorkflowRunEventV4( + { + runId: 'wrun_1', + eventType: 'wait_created', + specVersion: 5, + correlationId: 'wait_1', + }, + { token: 'test-token', dispatcher: agent } + ); + + expect('eventId' in (capturedMeta ?? {})).toBe(false); + expect('maxSlot' in (capturedMeta ?? {})).toBe(false); + agent.assertNoPendingInterceptors(); + }); }); /** diff --git a/packages/world-vercel/src/events-v4.ts b/packages/world-vercel/src/events-v4.ts index e0556e092b..292bfd7798 100644 --- a/packages/world-vercel/src/events-v4.ts +++ b/packages/world-vercel/src/events-v4.ts @@ -21,6 +21,7 @@ * bytes — this module stays at the wire-bytes layer. */ +import { SlotConflictError } from '@workflow/errors'; import { decode } from 'cbor-x'; import { decodeFrames, encodeFrame, V4_FRAME_CONTENT_TYPE } from './frames.js'; import { getEventsDispatcher } from './http-client.js'; @@ -72,13 +73,33 @@ async function fetchV4( errorFromV4Response( response.status, headersToRecord(response.headers), - await response.text(), + await readErrorBody(response), opName, url ), }); } +/** + * The error body as bytes when it is CBOR, as text otherwise. + * + * Most v4 error responses are JSON, because the client sends no `Accept` header + * and the backend's error encoder defaults to it. A slot conflict is the + * exception: its body carries the event-log delta the client needs, whose + * payloads are byte strings that JSON cannot represent, so the backend encodes + * that one as CBOR regardless of the `Accept` header. + */ +async function readErrorBody(response: Response): Promise { + if (isCborContentType(response.headers.get('content-type'))) { + return new Uint8Array(await response.arrayBuffer()); + } + return await response.text(); +} + +function isCborContentType(contentType: string | null | undefined): boolean { + return contentType?.toLowerCase().includes('application/cbor') ?? false; +} + /** Flatten a fetch `Headers` into the record shape throwForErrorResponse * expects (it mirrors the v3 `makeRequest` error contract). */ function headersToRecord(headers: Headers): Record { @@ -207,6 +228,22 @@ export interface CreateEventV4Input { * without a loaded event log; older servers ignore it entirely. */ stateUpdatedAt?: number; + /** + * The event's id, claimed by the client instead of minted by the server. + * Sent only for a run that numbers its events by slot, where the id encodes + * the event's position in the log. The server inserts it conditionally and + * answers 409 `slot-conflict` when the slot is already taken; a run on the + * older numbering that sends one is rejected with 400. Older servers ignore + * the field and mint an id as before — which is why only runs stamped with + * slot identity ever send it. + */ + eventId?: string; + /** + * The highest slot the client has seen in the run's event log (0 when it has + * seen none). Observability only: slots are dense, so a persisted slot more + * than one past this is a permanent hole in the log. Ignored by older servers. + */ + maxSlot?: number; } export interface CreateEventV4Result { @@ -305,9 +342,78 @@ function buildPostFrameMeta( if (input.stateUpdatedAt !== undefined) { meta.stateUpdatedAt = input.stateUpdatedAt; } + if (input.eventId !== undefined) meta.eventId = input.eventId; + if (input.maxSlot !== undefined) meta.maxSlot = input.maxSlot; return meta; } +/** + * The backend's machine-readable code for a lost event slot. Paired with 409 + * rather than 412 so a slot conflict stays distinguishable from the + * `stateUpdatedAt` watermark's staleness rejection while both are live. + */ +const V4_SLOT_CONFLICT_CODE = 'slot-conflict'; + +/** The fields a v4 error body may carry, whatever encoding it arrived in. */ +interface V4ErrorBody { + message?: unknown; + /** Machine-readable code. The backend names this field `error`. */ + error?: unknown; + code?: unknown; + events?: unknown; + cursor?: unknown; + hasMore?: unknown; + details?: unknown; +} + +/** Decode an error body as CBOR or JSON, or `undefined` if it is neither. */ +function decodeErrorBody( + errorBody: string | Uint8Array +): V4ErrorBody | undefined { + try { + const value = + typeof errorBody === 'string' + ? (JSON.parse(errorBody) as unknown) + : (decode(errorBody) as unknown); + return value && typeof value === 'object' + ? (value as V4ErrorBody) + : undefined; + } catch { + return undefined; + } +} + +/** + * Build the `SlotConflictError` for a 409 whose body names a taken slot. + * + * The conflicting event id comes from the response header rather than the body + * so the error is still actionable when the body failed to decode; the delta is + * best-effort in the other direction — an absent or malformed `events` leaves + * the runtime to reload the log itself, which is always correct. + */ +function slotConflictFromBody( + message: string, + responseHeaders: Record, + body: V4ErrorBody | undefined +): SlotConflictError { + const details = body?.details; + const detailEventId = + details && typeof details === 'object' && 'eventId' in details + ? (details as { eventId?: unknown }).eventId + : undefined; + const headerEventId = readHeader( + responseHeaders, + V4_RESPONSE_HEADERS.eventId + ); + return new SlotConflictError(message, { + eventId: + headerEventId ?? (typeof detailEventId === 'string' ? detailEventId : ''), + events: Array.isArray(body?.events) ? body.events : [], + cursor: typeof body?.cursor === 'string' ? body.cursor : null, + hasMore: body?.hasMore === true, + }); +} + /** * Build the typed error for a non-2xx v4 response. Reuses the shared * `errorForResponse` status → error-type contract (409→EntityConflictError, @@ -320,19 +426,27 @@ function buildPostFrameMeta( function errorFromV4Response( statusCode: number, responseHeaders: Record, - errorBody: string, + errorBody: string | Uint8Array, opName: string, url: string ): Error { let message = `v4 ${opName} failed: HTTP ${statusCode}`; let code: string | undefined; - try { - const json = JSON.parse(errorBody) as { message?: string; code?: string }; - if (typeof json.message === 'string') message = json.message; - if (typeof json.code === 'string') code = json.code; - } catch { - // body wasn't JSON — keep the default message, append raw text below - if (errorBody) message += ` ${errorBody}`; + const decoded = decodeErrorBody(errorBody); + if (decoded) { + if (typeof decoded.message === 'string') message = decoded.message; + if (typeof decoded.code === 'string') code = decoded.code; + } else if (typeof errorBody === 'string' && errorBody) { + // Body was neither JSON nor CBOR — keep the default message and append the + // raw text so the response is still diagnosable. + message += ` ${errorBody}`; + } + + // A lost event slot is the one 409 that is not an entity conflict. The + // backend names its machine-readable code `error`; that field is read only + // here, so every other error keeps the status → type mapping below unchanged. + if (statusCode === 409 && decoded?.error === V4_SLOT_CONFLICT_CODE) { + return slotConflictFromBody(message, responseHeaders, decoded); } const retryAfter = parseRetryAfter( @@ -355,7 +469,7 @@ function errorFromV4Response( export function throwForErrorResponse( statusCode: number, responseHeaders: Record, - errorBody: string, + errorBody: string | Uint8Array, opName: string, url: string ): never { diff --git a/packages/world-vercel/src/events.ts b/packages/world-vercel/src/events.ts index 3e13e301c6..4d03450837 100644 --- a/packages/world-vercel/src/events.ts +++ b/packages/world-vercel/src/events.ts @@ -32,7 +32,11 @@ * the v3 path. */ -import { HookNotFoundError, WorkflowWorldError } from '@workflow/errors'; +import { + HookNotFoundError, + SlotConflictError, + WorkflowWorldError, +} from '@workflow/errors'; import { type AnyEventRequest, type CreateEventParams, @@ -465,6 +469,35 @@ function coerceNormalizedEvent(raw: Record): Event { return coerceEventDates(normalizeEventData(raw)); } +/** + * Runs an event create, normalizing the event-log delta a slot conflict carries + * into the same `Event` shape every other read path produces. + * + * The delta arrives as raw CBOR off the error response, so its nested dates are + * still ISO strings; the runtime merges these events into its loaded log and + * calls `.getTime()` on them, exactly as it does for the inline delta on the + * success path. Doing the coercion here rather than in the runtime keeps the + * wire's shape a concern of this adapter, and keeps `SlotConflictError.events` + * meaning the same thing for every World that raises it. + */ +async function withCoercedSlotConflictDelta( + op: () => Promise +): Promise { + try { + return await op(); + } catch (error) { + if (!SlotConflictError.is(error) || error.events.length === 0) { + throw error; + } + throw new SlotConflictError(error.message, { + eventId: error.eventId, + events: (error.events as Record[]).map(coerceEventDates), + cursor: error.cursor, + hasMore: error.hasMore, + }); + } +} + function decodeLegacyStructuredError(payload: Uint8Array): unknown { if (hasSerializedDataFormatPrefix(payload)) { return payload; @@ -618,9 +651,11 @@ export async function createWorkflowRunEvent( // the next queue delivery. Non-retryable // types (step_started, step_retrying, hook_received) run once. See // ./event-retry for the validated per-event classification. - return await withEventPostRetry( - () => createWorkflowRunEventInner(id, data, params, config), - data.eventType + return await withCoercedSlotConflictDelta(() => + withEventPostRetry( + () => createWorkflowRunEventInner(id, data, params, config), + data.eventType + ) ); } catch (err) { // 404 on hook_disposed / hook_received → already-disposed hook. @@ -721,6 +756,13 @@ async function createWorkflowRunEventInner( // skip the list+resolve. The server only acts on it for run_started; // older servers ignore it and simply preload as before. ...(params?.skipPreload ? { skipPreload: true } : {}), + // Slot identity: the runtime names the event's own id, claiming that + // position in the run's event log. The server inserts it conditionally + // and answers 409 slot-conflict when another writer got there first. + // `maxSlot` rides along so the server can spot a gap, which slots being + // dense makes an unrecoverable corruption. + ...(params?.eventId ? { eventId: params.eventId } : {}), + ...(params?.maxSlot !== undefined ? { maxSlot: params.maxSlot } : {}), remoteRefBehavior, payload, ...meta, diff --git a/packages/world/src/events.ts b/packages/world/src/events.ts index 2a581f76bf..a8cb09c133 100644 --- a/packages/world/src/events.ts +++ b/packages/world/src/events.ts @@ -773,6 +773,37 @@ export interface CreateEventParams { * across the SDK and the backend. */ skipPreload?: boolean; + /** + * The event's id, chosen by the client rather than the World. + * + * Only sent for a run whose spec version numbers events by slot + * (`SPEC_VERSION_SLOT_IDENTITY`), where the id encodes the event's position in + * the log and is therefore the client's claim on that position. + * + * Backend contract (for World implementers who want to support slot + * identity): treat the id as a claim to be won, not a hint. Insert it under a + * uniqueness constraint on `(runId, eventId)` and, when the id is already + * taken, reject the write with `SlotConflictError` (HTTP 409) instead of + * minting a different id — a lost slot means the client replayed against an + * event log missing at least one event, so its whole proposed event, not just + * its id, is suspect. Reject a mismatch in either direction with a 400: an id + * of this shape on a run that does not use slot identity, or an absent or + * ULID-shaped id on a run that does, would leave the log unable to prove its + * own completeness. + * + * A World that ignores this field keeps minting ids itself, which is correct + * only for runs that were never stamped with slot identity in the first + * place. + */ + eventId?: string; + /** + * The highest slot the client has observed in the run's event log, or 0 for a + * log with no slot-numbered events. Sent alongside {@link eventId} purely as + * an observability signal: because slots are dense, a persisted slot more + * than one past this is a hole, which is unrecoverable and worth alerting on. + * Worlds MAY ignore it. + */ + maxSlot?: number; } /** diff --git a/packages/world/src/index.ts b/packages/world/src/index.ts index c6f601f36c..7317b340ba 100644 --- a/packages/world/src/index.ts +++ b/packages/world/src/index.ts @@ -104,7 +104,9 @@ export { export { FIRST_SLOT, isSlotId, + maxSlotOf, SLOT_ID_WIDTH, + slotEventId, slotFromId, slotIdBody, } from './slot-identity.js'; diff --git a/packages/world/src/slot-identity.test.ts b/packages/world/src/slot-identity.test.ts index 46c5e7d7d7..c99d2aa930 100644 --- a/packages/world/src/slot-identity.test.ts +++ b/packages/world/src/slot-identity.test.ts @@ -3,7 +3,9 @@ import { describe, expect, it } from 'vitest'; import { FIRST_SLOT, isSlotId, + maxSlotOf, SLOT_ID_WIDTH, + slotEventId, slotFromId, slotIdBody, } from './slot-identity.js'; @@ -57,3 +59,30 @@ describe('slotFromId', () => { expect(slotFromId(`evnt_${'1'.repeat(SLOT_ID_WIDTH + 1)}`)).toBeUndefined(); }); }); + +describe('maxSlotOf', () => { + it('finds the highest slot regardless of position', () => { + // A log is merged from several loads and is not sorted, so the last element + // is not necessarily the highest slot. + expect( + maxSlotOf([slotEventId(3), slotEventId(7), slotEventId(1)].map(toEvent)) + ).toBe(7); + }); + + it('reports 0 for an empty or ULID-numbered log', () => { + expect(maxSlotOf([])).toBe(0); + expect(maxSlotOf([toEvent(`evnt_${ulid()}`)])).toBe(0); + }); + + it('ignores ULID ids mixed in with slots', () => { + // A mixed log violates slot identity's purity invariant, but the scan must + // still report the highest slot rather than throwing or returning 0. + expect( + maxSlotOf([toEvent(`evnt_${ulid()}`), toEvent(slotEventId(2))]) + ).toBe(2); + }); +}); + +function toEvent(eventId: string): { eventId: string } { + return { eventId }; +} diff --git a/packages/world/src/slot-identity.ts b/packages/world/src/slot-identity.ts index c4d609c754..d9ff11b65f 100644 --- a/packages/world/src/slot-identity.ts +++ b/packages/world/src/slot-identity.ts @@ -61,3 +61,26 @@ export function slotFromId(id: string): number | undefined { export function isSlotId(id: string): boolean { return slotFromId(id) !== undefined; } + +/** The event id occupying `slot`. */ +export function slotEventId(slot: number): string { + return `evnt_${slotIdBody(slot)}`; +} + +/** + * The highest slot named by any of `events`, or 0 when none is slot-numbered. + * + * Scans rather than reading the last element: a log is merged from several + * loads and is not necessarily sorted, and callers use this value to pick the + * next free slot. + */ +export function maxSlotOf(events: readonly { eventId: string }[]): number { + let max = 0; + for (const event of events) { + const slot = slotFromId(event.eventId); + if (slot !== undefined && slot > max) { + max = slot; + } + } + return max; +} From f8c41a18b5e4ddff43d9e9f8970038148d7fe4f3 Mon Sep 17 00:00:00 2001 From: Peter Wielander Date: Thu, 30 Jul 2026 17:46:11 -0700 Subject: [PATCH 05/35] feat(worlds): number events by slot in the Local and Postgres Worlds Slot identity is only useful if a World can actually keep it, so both first-party Worlds now allocate, honour, and defend dense per-run positions: - `SPEC_VERSION_MAX_SUPPORTED` separates the newest version a World can read from the version it stamps, so turning the flag on somewhere does not make the runs it creates unreadable elsewhere. - `mintedSpecVersion()` gives both Worlds one place to opt new runs in. - The Local World allocates under its storage lock and re-probes on a lost exclusive write; the Postgres World makes the events primary key run-scoped and treats a unique violation as "try the next free position", re-probing every round so contention always makes progress. - A caller-claimed position that is already taken is a 409 carrying the events the caller was missing, and nothing is materialized for it. Co-Authored-By: Claude Opus 5 --- .changeset/slot-event-identity-worlds.md | 8 + .../docs/v5/configuration/runtime-tuning.mdx | 9 + packages/core/src/runtime.ts | 113 +++- packages/core/src/runtime/helpers.test.ts | 43 ++ packages/core/src/runtime/helpers.ts | 34 +- packages/core/src/runtime/start.test.ts | 33 +- .../core/src/runtime/world-compatibility.ts | 27 +- packages/core/src/workflow.ts | 26 +- packages/world-local/src/index.ts | 7 +- .../world-local/src/storage/events-storage.ts | 288 ++++++++- packages/world-local/src/storage/helpers.ts | 92 ++- .../src/storage/slot-identity.test.ts | 281 +++++++++ .../world-local/src/storage/slots.test.ts | 241 +++++++ packages/world-local/src/storage/slots.ts | 247 ++++++++ .../0018_run_scoped_event_and_step_keys.sql | 13 + .../src/drizzle/migrations/meta/_journal.json | 7 + packages/world-postgres/src/drizzle/schema.ts | 18 +- packages/world-postgres/src/index.ts | 7 +- packages/world-postgres/src/slots.ts | 225 +++++++ packages/world-postgres/src/storage.ts | 587 ++++++++++++------ .../world-postgres/test/slot-identity.test.ts | 375 +++++++++++ packages/world/src/index.ts | 3 + packages/world/src/slot-identity.test.ts | 25 +- packages/world/src/spec-version.test.ts | 64 +- packages/world/src/spec-version.ts | 52 +- packages/world/src/ulid.ts | 12 + 26 files changed, 2529 insertions(+), 308 deletions(-) create mode 100644 .changeset/slot-event-identity-worlds.md create mode 100644 packages/world-local/src/storage/slot-identity.test.ts create mode 100644 packages/world-local/src/storage/slots.test.ts create mode 100644 packages/world-local/src/storage/slots.ts create mode 100644 packages/world-postgres/src/drizzle/migrations/0018_run_scoped_event_and_step_keys.sql create mode 100644 packages/world-postgres/src/slots.ts create mode 100644 packages/world-postgres/test/slot-identity.test.ts diff --git a/.changeset/slot-event-identity-worlds.md b/.changeset/slot-event-identity-worlds.md new file mode 100644 index 0000000000..863a2f860b --- /dev/null +++ b/.changeset/slot-event-identity-worlds.md @@ -0,0 +1,8 @@ +--- +'@workflow/world': minor +'@workflow/world-local': minor +'@workflow/world-postgres': minor +'@workflow/core': minor +--- + +Number a run's events by position in the Local and Postgres Worlds when `WORKFLOW_SLOT_IDENTITY` is set, so a reader can prove its copy of an event log is complete. diff --git a/docs/content/docs/v5/configuration/runtime-tuning.mdx b/docs/content/docs/v5/configuration/runtime-tuning.mdx index 2f3e68e8c3..110e569d5f 100644 --- a/docs/content/docs/v5/configuration/runtime-tuning.mdx +++ b/docs/content/docs/v5/configuration/runtime-tuning.mdx @@ -48,6 +48,15 @@ For example, a workflow can run a 10-minute inline step even with `WORKFLOW_REPL - Backends that do not support the guard ignore the snapshot; they must not declare the capability, so guard-dependent optimizations stay off against them even when the flag is set. - Set `0` to disable. +### `WORKFLOW_SLOT_IDENTITY` + +- Default: disabled +- Numbers a new run's events by position instead of by ULID: `evnt_…001` is the run's first event, `evnt_…002` its second, with no gaps. A reader can then prove its copy of the log is complete, because the highest number is the event count. +- Contention becomes explicit rather than silent. Two writers proposing the same position cannot both win: the loser gets a 409 ([`SlotConflictError`](/docs/api-reference/workflow-errors/slot-conflict-error)) carrying the events it was missing, and the runtime merges them, replays, and re-proposes at the next free position. +- Applies only to runs created while it is set. A run keeps the identity scheme it was created with for life, so turning the flag on or off never affects runs already in flight. +- Requires a World that supports it. A World that does not rejects the run outright rather than mis-numbering its events. +- Set `1` or `true` to enable. + ## Inline execution ### `WORKFLOW_V2_TIMEOUT_MS` diff --git a/packages/core/src/runtime.ts b/packages/core/src/runtime.ts index 86a04e0e9c..eed5ba9f9f 100644 --- a/packages/core/src/runtime.ts +++ b/packages/core/src/runtime.ts @@ -4,7 +4,6 @@ import { EntityConflictError, FatalError, MaxEventsExceededError, - PreconditionFailedError, ReplayDivergenceError, RUN_ERROR_CODES, type RunErrorCode, @@ -24,6 +23,7 @@ import { resolveQueueNamespace, SPEC_VERSION_CURRENT, SPEC_VERSION_SUPPORTS_COMPRESSION, + slotFromId, WorkflowInvokePayloadSchema, type WorkflowRun, type World, @@ -57,6 +57,7 @@ import { memoizeEncryptionKey, parseHealthCheckPayload, queueMessage, + requiresFreshReplay, toMutableEventLog, withEventCreateFence, withHealthCheck, @@ -547,6 +548,19 @@ export function workflowEntrypoint( let workflowStartedAt = -1; let preloadedEvents: Event[] | undefined; let preloadedEventsCursor: string | null | undefined; + // Highest slot known to be published on a slot-numbered run, + // for the writes whose snapshot cannot show it: turbo + // backgrounds `run_started` and replays against an empty log, + // so a claim numbered from that log alone would propose a slot + // `run_started` already holds. 0 when the run is not + // slot-numbered — its ids carry no position to compare. + let knownSlotFloor = 0; + const observeSlotFloor = (eventId: string | undefined) => { + const slot = eventId ? slotFromId(eventId) : undefined; + if (slot !== undefined && slot > knownSlotFloor) { + knownSlotFloor = slot; + } + }; // Latency telemetry (TTFS) state — see runtime/step-latency.ts. // Whether this invocation's FIRST event snapshot contained @@ -1021,6 +1035,10 @@ export function workflowEntrypoint( (r) => { const limit = clampMaxEvents(r?.maxEvents); if (limit !== undefined) maxEventsLimit = limit; + // Every write of this invocation is ordered after this + // promise by `runReadyBarrier`, so the slot it reports + // is in hand before the first claim is numbered. + observeSlotFloor(r?.event?.eventId); }, () => {} ); @@ -1088,6 +1106,7 @@ export function workflowEntrypoint( } workflowRun = result.run; maxEventsLimit = clampMaxEvents(result.maxEvents); + observeSlotFloor(result.event?.eventId); // Anchors RSFS — see the declaration above. runStartedReceivedAtMs = Date.now(); @@ -1367,7 +1386,11 @@ export function workflowEntrypoint( // place by the guard's reloads, so a per-iteration // rescan for the slot high-water mark would be wasted // work on an array that never changes identity here. - const waitLog = toMutableEventLog(events, eventsCursor); + const waitLog = toMutableEventLog( + events, + eventsCursor, + knownSlotFloor + ); for (const waitEvent of waitsToComplete) { try { await withEventCreateFence( @@ -1491,6 +1514,20 @@ export function workflowEntrypoint( // point and the inline executeStep mutates eventsCursor. preInlineWriteCursor = eventsCursor; + // One log for everything this replay writes on its way to + // a terminal event: the end-of-run drain inside + // `runWorkflow` (fire-and-forget `*_created` events, and + // the implicit disposal of the abort hooks a completing + // run leaves behind) and the `run_completed` / + // `run_failed` write below. Sharing it is what keeps the + // two from claiming the same slot — the terminal write + // numbers from a snapshot that predates the drain. + const replayWriteLog = toMutableEventLog( + events, + eventsCursor, + knownSlotFloor + ); + // Replay workflow runtimeLogger.debug('Starting workflow replay', { workflowRunId: runId, @@ -1516,7 +1553,8 @@ export function workflowEntrypoint( // `awaitRunReady()` below, so gate those writes on the // backgrounded run_started too. Undefined outside turbo. runReadyBarrier, - world.capabilities + world.capabilities, + replayWriteLog ); await payloadPrewarm; runtimeLogger.debug('Workflow replay completed', { @@ -1527,11 +1565,12 @@ export function workflowEntrypoint( // Workflow completed. Send the snapshot but do NOT // reload-and-retry the create in place: `result` was - // computed by this replay, so a stale (412) rejection must - // force a *fresh replay* (which may observe the new event - // and produce a different result), not re-commit the stale - // result. The catch below lets PreconditionFailedError - // propagate to the queue for re-invocation. + // computed by this replay, so a rejection proving the view + // was incomplete must force a *fresh replay* (which may + // observe the new event and produce a different result), + // not re-commit the stale result. The catch below lets + // `requiresFreshReplay` rejections propagate to the queue + // for re-invocation. try { // Turbo: a workflow that finishes with no steps reaches // here before the backgrounded run_started; order the @@ -1547,7 +1586,7 @@ export function workflowEntrypoint( { requestId, ...eventCreateFenceFor( - toMutableEventLog(events, eventsCursor), + replayWriteLog, workflowRun.specVersion ), } @@ -1637,7 +1676,8 @@ export function workflowEntrypoint( } const suspensionLog = toMutableEventLog( cachedEvents, - eventsCursor + eventsCursor, + knownSlotFloor ); let suspensionResult: Awaited< ReturnType @@ -1653,13 +1693,14 @@ export function workflowEntrypoint( runReadyBarrier, }); } catch (suspensionError) { - // A suspension create whose stale (412) rejection - // survived the in-guard reload retries: schedule an + // A suspension create whose incomplete-view rejection + // (412 stale watermark, or 409 taken slot) survived + // the in-guard reload retries: schedule an // explicit immediate re-invocation (a rethrow relies // on redelivery of a message the turbo path already // acked — the run would stall for the queue's ~300s // default visibility timeout). - if (PreconditionFailedError.is(suspensionError)) { + if (requiresFreshReplay(suspensionError)) { runtimeLogger.warn( 'Suspension event creation rejected as stale after reload retries; re-invoking run for a fresh replay', { workflowRunId: runId, loopIteration } @@ -2273,10 +2314,14 @@ export function workflowEntrypoint( // whole batch so each claim draws its own event slot; // `eventCreateFenceFor` yields undefined for a run // fenced neither way, leaving those claims as they were. - const inlineClaimLog = toMutableEventLog( - cachedEvents ?? [], - eventsCursor - ); + // + // The suspension's own log, not a second one over the + // same snapshot: its reservations are what the hook and + // wait creates just above took, and those events are not + // in `cachedEvents` yet. A fresh log would number these + // claims from the same base and hand the batch's first + // step a slot the suspension already holds. + const inlineClaimLog = suspensionLog; replayBudget.pause(); let stepResults: Awaited< @@ -2405,17 +2450,18 @@ export function workflowEntrypoint( stepExecutionPromises ); } catch (stepErr) { - // A stale (412) rejection of an inline step_started - // claim: the loaded view this batch was scheduled - // from is behind an out-of-band event (e.g. a - // received hook), so the claim was fenced by the + // An incomplete-view rejection of an inline + // step_started claim (412 stale watermark, or 409 + // taken slot): the loaded view this batch was + // scheduled from is behind an out-of-band event (e.g. + // a received hook), so the claim was fenced by the // guard and no step events were written. Abandon the // batch — any optimistic body result is discarded by // executeStep's reconciliation — and re-invoke for a // fresh replay that observes the new event. Wait for // the sibling executions to settle first so no owned // body is in flight when the ack path runs. - if (PreconditionFailedError.is(stepErr)) { + if (requiresFreshReplay(stepErr)) { await Promise.allSettled(stepExecutionPromises); runtimeLogger.warn( 'Inline step claim rejected as stale; re-invoking run for a fresh replay', @@ -2612,17 +2658,18 @@ export function workflowEntrypoint( } } } else { - // Stale-snapshot rejection of a result-bearing create - // (run_completed sends the snapshot but is intentionally - // NOT retried in place), or one that survived the - // in-guard reload retries. Don't fail the run — schedule - // an explicit immediate re-invocation so a fresh replay - // observes the new event. Rethrowing instead would rely - // on redelivery of the CURRENT message, which the turbo - // path has already acked — empirically the run then - // stalls for the queue's ~300s default visibility - // timeout before completing. - if (PreconditionFailedError.is(err)) { + // Incomplete-view rejection of a result-bearing create — + // a stale watermark (412) or a taken slot (409), either + // on `run_completed` (which sends its fence but is + // intentionally NOT retried in place) or on a create + // that survived the in-guard reload retries. Don't fail + // the run — schedule an explicit immediate re-invocation + // so a fresh replay observes the new event. Rethrowing + // instead would rely on redelivery of the CURRENT + // message, which the turbo path has already acked — + // empirically the run then stalls for the queue's ~300s + // default visibility timeout before completing. + if (requiresFreshReplay(err)) { runtimeLogger.warn( 'Event creation rejected as stale; re-invoking run for a fresh replay', { workflowRunId: runId, loopIteration } diff --git a/packages/core/src/runtime/helpers.test.ts b/packages/core/src/runtime/helpers.test.ts index 15264d404c..c62f23d735 100644 --- a/packages/core/src/runtime/helpers.test.ts +++ b/packages/core/src/runtime/helpers.test.ts @@ -1,4 +1,5 @@ import { + EntityConflictError, PreconditionFailedError, SlotConflictError, WorkflowWorldError, @@ -29,6 +30,7 @@ import { memoizeEncryptionKey, mergeLoadedEvents, PRECONDITION_MAX_RELOAD_RETRIES, + requiresFreshReplay, reserveSlot, stateUpdatedAtForCreate, toMutableEventLog, @@ -537,6 +539,26 @@ describe('slot bookkeeping', () => { ).toBe(0); }); + it('starts at a floor the snapshot cannot show', () => { + // Turbo replays against an empty log while its `run_started` write is still + // in flight, so the snapshot alone would number the first claim onto a slot + // that write already holds. + const log = toMutableEventLog([], null, 2); + expect(log.maxSlot).toBe(2); + expect(reserveSlot(log)).toBe(3); + }); + + it('ignores a floor the snapshot has already passed', () => { + const log = toMutableEventLog([slotEvent(5)], 'c0', 2); + expect(log.maxSlot).toBe(5); + }); + + it('keeps the floor across a merge', () => { + const log = toMutableEventLog([], null, 2); + mergeLoadedEvents(log, [slotEvent(1)]); + expect(log.maxSlot).toBe(2); + }); + it('never lowers maxSlot when an older delta is merged in', () => { const log = toMutableEventLog([slotEvent(1), slotEvent(3)], 'c0'); mergeLoadedEvents(log, [slotEvent(2)]); @@ -810,6 +832,27 @@ describe('withEventCreateFence', () => { }); }); +describe('requiresFreshReplay', () => { + it('covers both fences, so neither numbering fails the run', () => { + // Each fence reports an incomplete view in its own dialect. A caller that + // recognises only one of them fails runs on the other. + expect(requiresFreshReplay(new PreconditionFailedError('stale'))).toBe( + true + ); + expect( + requiresFreshReplay( + new SlotConflictError('taken', { eventId: slotEventId(3) }) + ) + ).toBe(true); + }); + + it('leaves every other rejection to its own handler', () => { + expect(requiresFreshReplay(new EntityConflictError('exists'))).toBe(false); + expect(requiresFreshReplay(new Error('boom'))).toBe(false); + expect(requiresFreshReplay(undefined)).toBe(false); + }); +}); + describe('withPreconditionRetry', () => { let originalGuard: string | undefined; diff --git a/packages/core/src/runtime/helpers.ts b/packages/core/src/runtime/helpers.ts index 0dda520104..6720f22cd7 100644 --- a/packages/core/src/runtime/helpers.ts +++ b/packages/core/src/runtime/helpers.ts @@ -669,12 +669,26 @@ export interface MutableEventLog { reserved: number; } -/** A `MutableEventLog` over a freshly loaded snapshot. */ +/** + * A `MutableEventLog` over a freshly loaded snapshot. + * + * `slotFloor` is a slot known to be published that the snapshot may not contain + * — the run's own `run_started`, whose write turbo backgrounds while replaying + * against an empty log. Numbering a claim from the snapshot alone would then + * propose a slot that is already taken, so every first write of a turbo + * invocation would conflict and cost the run an extra replay. + */ export function toMutableEventLog( events: Event[], - cursor: string | null + cursor: string | null, + slotFloor = 0 ): MutableEventLog { - return { events, cursor, maxSlot: maxSlotOf(events), reserved: 0 }; + return { + events, + cursor, + maxSlot: Math.max(maxSlotOf(events), slotFloor), + reserved: 0, + }; } /** @@ -962,6 +976,20 @@ export function withEventCreateFence( ); } +/** + * Whether a rejected event create means "this replay's view of the log was + * incomplete", the one condition whose only remedy is replaying from the top. + * + * Both fences report it, one per numbering: a 412 says the snapshot's watermark + * is behind, a 409 says the slot this replay counted to is already occupied. + * Neither is a failure of the run — the run's own decisions may simply need + * revising against the events it did not see — so a caller that gets one + * re-invokes for a fresh replay rather than failing. + */ +export function requiresFreshReplay(error: unknown): boolean { + return PreconditionFailedError.is(error) || SlotConflictError.is(error); +} + /** * CORS headers for health check responses. * Allows the observability UI to check endpoint health from a different origin. diff --git a/packages/core/src/runtime/start.test.ts b/packages/core/src/runtime/start.test.ts index 29a7c80419..d04a415bb7 100644 --- a/packages/core/src/runtime/start.test.ts +++ b/packages/core/src/runtime/start.test.ts @@ -2,6 +2,7 @@ import { WorkflowRuntimeError, WorkflowWorldError } from '@workflow/errors'; import { SPEC_VERSION_CURRENT, SPEC_VERSION_LEGACY, + SPEC_VERSION_MAX_SUPPORTED, SPEC_VERSION_SUPPORTS_ATTRIBUTES, SPEC_VERSION_SUPPORTS_CBOR_QUEUE_TRANSPORT, } from '@workflow/world'; @@ -136,7 +137,7 @@ describe('start', () => { } as any); await expect(start(validWorkflow, [])).rejects.toThrow( - 'requires a World with matching spec version' + 'requires a World with spec version' ); expect(mockEventsCreate).not.toHaveBeenCalled(); expect(mockQueue).not.toHaveBeenCalled(); @@ -174,7 +175,7 @@ describe('start', () => { } as any); await expect(start(validWorkflow, [])).rejects.toThrow( - 'requires a World with matching spec version' + 'requires a World with spec version' ); expect(mockEventsCreate).not.toHaveBeenCalled(); expect(mockQueue).not.toHaveBeenCalled(); @@ -186,19 +187,43 @@ describe('start', () => { }); setWorld({ - specVersion: SPEC_VERSION_CURRENT + 1, + specVersion: SPEC_VERSION_MAX_SUPPORTED + 1, getDeploymentId: vi.fn().mockResolvedValue('deploy_123'), events: { create: mockEventsCreate }, queue: mockQueue, } as any); await expect(start(validWorkflow, [])).rejects.toThrow( - 'requires a World with matching spec version' + 'requires a World with spec version' ); expect(mockEventsCreate).not.toHaveBeenCalled(); expect(mockQueue).not.toHaveBeenCalled(); }); + it('accepts a world that mints a newer version this runtime supports', async () => { + // A world opted into slot identity stamps a version above the runtime's + // current one. The runtime can read and write those runs, so the + // handshake has to pass and the run has to keep the world's version. + const validWorkflow = Object.assign(() => Promise.resolve('result'), { + workflowId: 'test-workflow', + }); + + setWorld({ + specVersion: SPEC_VERSION_MAX_SUPPORTED, + getDeploymentId: vi.fn().mockResolvedValue('deploy_123'), + events: { create: mockEventsCreate }, + queue: mockQueue, + } as any); + + await start(validWorkflow, []); + + expect(mockEventsCreate).toHaveBeenCalledWith( + expect.stringMatching(/^wrun_/), + expect.objectContaining({ specVersion: SPEC_VERSION_MAX_SUPPORTED }), + expect.anything() + ); + }); + it('should use provided specVersion when passed in options', async () => { const validWorkflow = Object.assign(() => Promise.resolve('result'), { workflowId: 'test-workflow', diff --git a/packages/core/src/runtime/world-compatibility.ts b/packages/core/src/runtime/world-compatibility.ts index c06c26eefb..f3de904359 100644 --- a/packages/core/src/runtime/world-compatibility.ts +++ b/packages/core/src/runtime/world-compatibility.ts @@ -1,19 +1,40 @@ import { WorkflowRuntimeError } from '@workflow/errors'; import type { World } from '@workflow/world'; -import { SPEC_VERSION_CURRENT } from '@workflow/world'; +import { + SPEC_VERSION_CURRENT, + SPEC_VERSION_MAX_SUPPORTED, +} from '@workflow/world'; type WorldSpecVersionMetadata = Pick; +/** + * Rejects a World whose protocol this runtime does not speak. + * + * A World declares the spec version it stamps on the runs it creates. Anything + * from {@link SPEC_VERSION_CURRENT} up to {@link SPEC_VERSION_MAX_SUPPORTED} is + * fine: the upper end covers a World opted into a newer identity scheme that + * this runtime already understands, and only versions this runtime has no code + * for are refused. Below the current version means the World package predates + * this runtime and cannot record what it emits. + */ export function assertWorldSupportsRuntimeProtocol( world: WorldSpecVersionMetadata ): void { - if (world.specVersion === SPEC_VERSION_CURRENT) { + if ( + world.specVersion !== undefined && + world.specVersion >= SPEC_VERSION_CURRENT && + world.specVersion <= SPEC_VERSION_MAX_SUPPORTED + ) { return; } const supportedVersion = world.specVersion ?? 'none'; + const supported = + SPEC_VERSION_CURRENT === SPEC_VERSION_MAX_SUPPORTED + ? `${SPEC_VERSION_CURRENT}` + : `${SPEC_VERSION_CURRENT} to ${SPEC_VERSION_MAX_SUPPORTED}`; throw new WorkflowRuntimeError( - `This Workflow runtime requires a World with matching spec version ${SPEC_VERSION_CURRENT}, ` + + `This Workflow runtime requires a World with spec version ${supported}, ` + `but the configured World declares spec version ${supportedVersion}. ` + 'Install a World package version compatible with the current Workflow runtime.' ); diff --git a/packages/core/src/workflow.ts b/packages/core/src/workflow.ts index fde4b70250..97c9acee99 100644 --- a/packages/core/src/workflow.ts +++ b/packages/core/src/workflow.ts @@ -18,6 +18,7 @@ import { runtimeLogger } from './logger.js'; import type { WorkflowOrchestratorContext } from './private.js'; import { ReplayPayloadCache } from './replay-payload-cache.js'; import { getPortLazy } from './runtime/get-port-lazy.js'; +import type { MutableEventLog } from './runtime/helpers.js'; import { runIdCreatedAt } from './runtime/run-id-time.js'; import { handleSuspension } from './runtime/suspension-handler.js'; import { getWorld } from './runtime/world.js'; @@ -74,7 +75,15 @@ async function drainPendingQueueItems( * In turbo mode, gates final `*_created` writes on backgrounded * `run_started`. Undefined when `run_started` is awaited. */ - runReadyBarrier?: Promise + runReadyBarrier?: Promise, + /** + * The replay's event log, so the drain's writes claim their slots from the + * same source the terminal `run_completed` / `run_failed` write draws from. + * Without it the drain writes unfenced — the World picks the next free slot — + * and the terminal write, numbering from a snapshot taken before the drain, + * proposes the slot the drain just took and loses it. + */ + eventLog?: MutableEventLog ): Promise { if (pendingQueue.size === 0) return; // Implicitly dispose any abort hooks (system hooks) that are still alive at @@ -101,6 +110,7 @@ async function drainPendingQueueItems( world, run: workflowRun, runReadyBarrier, + eventLog, }); } catch (err) { runtimeLogger.warn( @@ -137,7 +147,13 @@ export async function runWorkflow( * Features supported by the World executing this workflow. Missing * capabilities are treated as unsupported. */ - worldCapabilities?: WorldCapabilities + worldCapabilities?: WorldCapabilities, + /** + * The caller's event log for this replay. Its only use here is the end-of-run + * drain, whose writes have to be ordered with the caller's terminal write — + * see {@link drainPendingQueueItems}. + */ + eventLog?: MutableEventLog ): Promise { return trace(`workflow.run ${workflowRun.workflowName}`, async (span) => { span?.setAttributes({ @@ -859,7 +875,8 @@ export async function runWorkflow( vmGlobalThis, workflowRun, 'completed', - runReadyBarrier + runReadyBarrier, + eventLog ); return dehydrated; @@ -876,7 +893,8 @@ export async function runWorkflow( vmGlobalThis, workflowRun, 'failed', - runReadyBarrier + runReadyBarrier, + eventLog ); throw err; diff --git a/packages/world-local/src/index.ts b/packages/world-local/src/index.ts index f8e2166fb7..2014a75652 100644 --- a/packages/world-local/src/index.ts +++ b/packages/world-local/src/index.ts @@ -2,7 +2,7 @@ import { promises as fs } from 'node:fs'; import { rm } from 'node:fs/promises'; import path from 'node:path'; import type { QueuePrefix, World } from '@workflow/world'; -import { reenqueueActiveRuns, SPEC_VERSION_CURRENT } from '@workflow/world'; +import { mintedSpecVersion, reenqueueActiveRuns } from '@workflow/world'; import { warnIfRunningInVercelDeployment } from './build-target-mismatch.js'; import type { Config } from './config.js'; import { config, resolveRecoverActiveRuns } from './config.js'; @@ -72,7 +72,10 @@ export function createWorld(args?: Partial): LocalWorld { ); const recoverActiveRuns = resolveRecoverActiveRuns(mergedConfig); return { - specVersion: SPEC_VERSION_CURRENT, + // What this world stamps on new runs, which is not the newest version it + // can read: slot identity is readable everywhere and minted only where + // WORKFLOW_SLOT_IDENTITY is set. + specVersion: mintedSpecVersion(), ...queue, ...storage, ...instrumentObject('world.streams', { diff --git a/packages/world-local/src/storage/events-storage.ts b/packages/world-local/src/storage/events-storage.ts index 7efd49cb1e..98e705e98d 100644 --- a/packages/world-local/src/storage/events-storage.ts +++ b/packages/world-local/src/storage/events-storage.ts @@ -5,11 +5,13 @@ import { HookNotFoundError, RunExpiredError, RunNotSupportedError, + SlotConflictError, TooEarlyError, WorkflowRunNotFoundError, WorkflowWorldError, } from '@workflow/errors'; import type { + CreateEventParams, Event, EventResult, Hook, @@ -34,8 +36,12 @@ import { isTerminalWorkflowRunStatus, requiresNewerWorld, SPEC_VERSION_CURRENT, + SPEC_VERSION_MAX_SUPPORTED, StepSchema, + slotEventId, + slotFromId, ulidToDate, + usesSlotIdentity, validateAttributeChanges, validateUlidTimestamp, WaitSchema, @@ -85,6 +91,7 @@ import { } from './hooks-storage.js'; import { handleLegacyEvent } from './legacy.js'; import { withRunFileLock } from './runs-storage.js'; +import { createSlotBook, RUN_CREATED_SLOT } from './slots.js'; /** * Per-run event ceiling the Local World reports on run responses (mirrors the @@ -506,6 +513,8 @@ export function createEventsStorage( const cachedPathsByRunId = new Map>(); let totalCachedEventBytes = 0; + const slots = createSlotBook(basedir, tag); + function deleteCachedEvent(eventPath: string): void { const event = eventCache.get(eventPath); if (!event) { @@ -525,6 +534,7 @@ export function createEventsStorage( for (const cachedPath of cachedPathsByRunId.get(runId) ?? []) { deleteCachedEvent(cachedPath); } + slots.forget(runId); } function clearCache(): void { @@ -532,6 +542,7 @@ export function createEventsStorage( cachedEventBytes.clear(); cachedPathsByRunId.clear(); totalCachedEventBytes = 0; + slots.clear(); } function cacheEvent( @@ -592,6 +603,64 @@ export function createEventsStorage( } } + /** + * The events a caller that just lost a slot is missing: one ascending page of + * the run's log after the cursor it wrote from, minus anything at or below the + * highest slot it already held. Because slots are dense, that second filter is + * exact — a caller cannot be missing an event whose position it can name. + * + * Returned inline with the conflict so the common case (a handful of events + * arrived out of band) costs the caller no extra round-trip. `hasMore` is + * forwarded verbatim: an overflowing delta is the caller's signal to page from + * `cursor` instead of treating this as the whole story. + */ + async function eventsAfterClaim( + runId: string, + params: CreateEventParams | undefined + ): Promise<{ events: Event[]; cursor: string | null; hasMore: boolean }> { + const page = await paginatedFileSystemQuery({ + directory: path.join(basedir, 'events'), + schema: EventSchema, + cachedItems: eventCache, + filePrefix: `${runId}-`, + sortOrder: 'asc', + ...(typeof params?.sinceCursor === 'string' + ? { cursor: params.sinceCursor } + : {}), + getCreatedAt: getObjectCreatedAt('evnt'), + getId: (event) => event.eventId, + }); + const maxSlot = params?.maxSlot ?? 0; + const resolveData = params?.resolveData ?? DEFAULT_RESOLVE_DATA_OPTION; + const missing = page.data.filter( + (event) => (slotFromId(event.eventId) ?? 0) > maxSlot + ); + return { + events: + resolveData === 'none' + ? missing.map((event) => stripEventDataRefs(event, resolveData)) + : missing, + cursor: page.cursor, + hasMore: page.hasMore, + }; + } + + /** + * The 409 a caller gets when the slot it named turns out to belong to someone + * else, carrying the events it is missing so it can merge, replay and + * re-propose at a free position. + */ + async function slotConflict( + runId: string, + eventId: string, + params: CreateEventParams | undefined + ): Promise { + return new SlotConflictError( + `Slot ${slotFromId(eventId)} of run "${runId}" is already taken`, + { eventId, ...(await eventsAfterClaim(runId, params)) } + ); + } + async function storeEvent(event: Event): Promise { const eventPath = taggedPath( basedir, @@ -643,6 +712,36 @@ export function createEventsStorage( if ('correlationId' in data && typeof data.correlationId === 'string') { assertSafeEntityId('correlationId', data.correlationId); } + if (params?.eventId !== undefined) { + assertSafeEntityId('eventId', params.eventId); + } + + // A slot-numbered create reserves its position before running the + // validation and materialization that may still reject it. Handing the + // reservation back on the way out is what keeps the log dense: an + // abandoned slot below a sibling's published one is a hole that can never + // be filled, and a log with a hole can no longer prove it is complete. + const reserved = new Set(); + let reservedRunId: string | undefined; + /** + * Hands the slots of a create that never published back to the allocator, + * so an abandoned reservation below a sibling's published slot does not + * become a hole the run can never fill. + */ + async function releasingSlots( + result: Promise + ): Promise { + try { + return await result; + } catch (error) { + if (reservedRunId !== undefined) { + for (const slot of reserved) { + slots.release(reservedRunId, slot); + } + } + throw error; + } + } // Step lifecycle events are serialized per-step via an in-process mutex // so that the "check state, then write" sequence in step_started / @@ -653,7 +752,9 @@ export function createEventsStorage( const lockKey = tag ? `${runId}-${data.correlationId}.${tag}` : `${runId}-${data.correlationId}`; - return withInProcessLock(stepLocks, lockKey, () => createImpl()); + return releasingSlots( + withInProcessLock(stepLocks, lockKey, () => createImpl()) + ); } // `hook_created` is serialized per-(runId, hookId) so the // "claim token, write hook entity, write event" sequence runs to @@ -682,9 +783,11 @@ export function createEventsStorage( const lockKey = tag ? `${runId}-${data.correlationId}.hook.${tag}` : `${runId}-${data.correlationId}.hook`; - return withInProcessLock(hookLocks, lockKey, () => createImpl()); + return releasingSlots( + withInProcessLock(hookLocks, lockKey, () => createImpl()) + ); } - return createImpl(); + return releasingSlots(createImpl()); async function createImpl(): Promise { // Most paths use the freshly-generated candidate eventId. The @@ -719,6 +822,18 @@ export function createEventsStorage( // specVersion is always sent by the runtime, but we provide a fallback for safety const effectiveSpecVersion = data.specVersion ?? SPEC_VERSION_CURRENT; + // Whether this run numbers its events by slot. Read from what was + // persisted, never from this request or this build, so a run stays in + // the mode it was created in for life — a run whose log holds ULID ids + // must never be handed a slot id, and vice versa. `run_created` is the + // one event that decides the mode instead of reading it; the + // resilient-start path below decides it too, on the request that + // creates the run. + let slotMode = + data.eventType === 'run_created' + ? usesSlotIdentity(effectiveSpecVersion) + : await slots.usesSlots(effectiveRunId); + // Get current run state for validation (if not creating a new run) // Skip run validation for step_completed and step_retrying - they only operate // on running steps, and running steps are always allowed to modify regardless @@ -800,8 +915,14 @@ export function createEventsStorage( ); if (created) { - // We created the run — also write the run_created event. - const runCreatedEventId = `evnt_${monotonicUlid()}`; + // We created the run, so this request also decided its mode. + slotMode = usesSlotIdentity(effectiveSpecVersion); + // We created the run — also write the run_created event. Its + // slot needs no allocation: a run's own `run_created` provably + // has nothing before it. + const runCreatedEventId = slotMode + ? slotEventId(RUN_CREATED_SLOT) + : `evnt_${monotonicUlid()}`; const runCreatedEvent: Event = { eventType: 'run_created', runId: effectiveRunId, @@ -820,6 +941,7 @@ export function createEventsStorage( }, }; await storeEvent(runCreatedEvent); + slots.observe(effectiveRunId, runCreatedEventId); currentRun = createdRun; } else { // Run already exists (concurrent run_created won the @@ -836,6 +958,17 @@ export function createEventsStorage( } } + // The run entity we just read is the authority on the mode, and it can + // appear between the probe above and this read: start() issues + // `run_created` and the queue send concurrently, so the delivery's + // `run_started` can arrive while the run is still being published. A + // stale "no" there would number that one event with a ULID on an + // otherwise slot-numbered run, and the hole it leaves in the numbering + // costs the log its completeness proof for life. + if (currentRun && data.eventType !== 'run_created') { + slotMode = usesSlotIdentity(currentRun.specVersion); + } + // run_failed on a non-existent run is rejected to match the // postgres and vercel worlds, which both surface this as a // WorkflowRunNotFoundError rather than silently persisting an @@ -857,7 +990,7 @@ export function createEventsStorage( if (requiresNewerWorld(currentRun.specVersion)) { throw new RunNotSupportedError( currentRun.specVersion!, - SPEC_VERSION_CURRENT + SPEC_VERSION_MAX_SUPPORTED ); } @@ -873,6 +1006,55 @@ export function createEventsStorage( } } + // ============================================================ + // EVENT ID: the caller's slot claim, an allocated slot, or a ULID + // ============================================================ + // A slot-numbered run's ids name positions in its log, so an id is + // either claimed by a caller that holds the log (and is therefore + // asserting the log is complete up to that position) or allocated here + // for a caller that has no log — a step completion reporting in, a + // cancellation from an API call. + if (params?.eventId !== undefined) { + const claimedSlot = slotFromId(params.eventId); + if (!slotMode) { + throw new WorkflowWorldError( + `Event id "${params.eventId}" was supplied for run "${effectiveRunId}", whose events are not numbered by slot`, + { status: 400 } + ); + } + if (claimedSlot === undefined) { + throw new WorkflowWorldError( + `Event id "${params.eventId}" is not a slot id, and run "${effectiveRunId}" numbers its events by slot`, + { status: 400 } + ); + } + eventId = params.eventId; + reservedRunId = effectiveRunId; + reserved.add(claimedSlot); + slots.claim(effectiveRunId, claimedSlot); + if (await slots.isWritten(effectiveRunId, claimedSlot)) { + // Reject a doomed claim before the materialization below creates + // the step, hook or wait this event will now never accompany. A + // caller that re-proposes at the next slot would otherwise + // collide with its own orphan and read that as "my write already + // landed". See SlotBook.isWritten. + throw await slotConflict(effectiveRunId, eventId, params); + } + } else if (slotMode) { + reservedRunId = effectiveRunId; + // A run's own `run_created` owns the first slot — provably, since + // nothing precedes it — so every other event allocates above it, even + // when that event is the first to arrive here. + const slot = await slots.reserve( + effectiveRunId, + data.eventType === 'run_created' + ? RUN_CREATED_SLOT + : RUN_CREATED_SLOT + 1 + ); + reserved.add(slot); + eventId = slotEventId(slot); + } + // ============================================================ // VALIDATION: Terminal state and event ordering checks // ============================================================ @@ -1127,13 +1309,40 @@ export function createEventsStorage( // strictly dominates all visible events of the run guarantees the // terminal event replays last. See mintRunDominantEventKey for // the dominance argument. - const dominantKey = await mintRunDominantEventKey( - basedir, - effectiveRunId, - tag - ); - eventId = dominantKey.eventId; - event = { ...event, eventId, createdAt: dominantKey.createdAt }; + // + // A key the *caller* chose is left alone. Its slot was picked from + // the caller's own log, so a concurrent event either sits below it + // (and already replays first) or takes the slot itself — in which + // case the publish below conflicts and the caller merges and + // re-proposes, which is the stronger answer. Re-numbering it here + // would also be actively wrong: the caller reserves slots for a + // whole flush of concurrent ops at once, so moving this one to + // "highest visible + 1" would steal the slot a sibling op is still + // in flight with. + if (params?.eventId === undefined) { + const dominantKey = await mintRunDominantEventKey( + basedir, + effectiveRunId, + tag, + slotMode + ); + const staleSlot = slotFromId(eventId); + const dominantSlot = slotFromId(dominantKey.eventId); + if (staleSlot !== undefined && staleSlot !== dominantSlot) { + // Only reachable when the log moved under us, which means the + // slot we held is now someone else's written event — handing it + // back leaves no hole. + reserved.delete(staleSlot); + slots.release(effectiveRunId, staleSlot); + } + if (dominantSlot !== undefined) { + reservedRunId = effectiveRunId; + reserved.add(dominantSlot); + slots.claim(effectiveRunId, dominantSlot); + } + eventId = dominantKey.eventId; + event = { ...event, eventId, createdAt: dominantKey.createdAt }; + } } // Create/update entity based on event type (event-sourced architecture) @@ -1505,13 +1714,19 @@ export function createEventsStorage( ); // Write the synthetic step_created event so replay observes it // (the client step consumer sets hasCreatedEvent only on a - // step_created event). Its eventId is a fresh monotonic ULID. + // step_created event). Its eventId is a second slot, or a fresh + // monotonic ULID — one request, two events. // Ordering vs. the step_started event row does not affect // correctness: the step_started consumer is a no-op and only // step_created flips hasCreatedEvent, so the end state is the // same whichever sorts first — this matches the resilient // run_started → run_created precedent in this file. - const stepCreatedEventId = `evnt_${monotonicUlid()}`; + let stepCreatedEventId = `evnt_${monotonicUlid()}`; + if (slotMode) { + const slot = await slots.reserve(effectiveRunId); + reserved.add(slot); + stepCreatedEventId = slotEventId(slot); + } const stepCreatedEvent: Event = { eventType: 'step_created', runId: effectiveRunId, @@ -1533,6 +1748,7 @@ export function createEventsStorage( ), stepCreatedEvent ); + slots.observe(effectiveRunId, stepCreatedEventId); validatedStep = createdStep; stepCreatedLazily = true; } @@ -2215,11 +2431,17 @@ export function createEventsStorage( // race here; whoever links the file first wins, the loser // throws EntityConflictError, and the runtime's existing // concurrent-replay catch path at suspension-handler.ts:142 - // swallows it. For all other event types, eventIds are - // monotonic ULIDs (globally unique by construction) so a - // collision indicates a real bug and EntityConflictError is + // swallows it. For all other event types of a ULID-numbered run, + // eventIds are monotonic ULIDs (globally unique by construction) so + // a collision indicates a real bug and EntityConflictError is // also the right surface — same shape as step_created's // claim-file behavior. + // + // A slot-numbered run collides by design: the id names a position in + // the log, so a loser is not a bug but a writer whose log was missing + // an event. It gets a SlotConflictError carrying that event instead + // (see below), and this write is the authority that decides it — the + // allocator's book is only ever a hint. // Last-instant re-validation for `hook_received` (see the acceptance // check above). The per-hook in-process lock already serializes // resume vs. dispose within one storage instance; this second check @@ -2311,10 +2533,15 @@ export function createEventsStorage( ); const staged = await writeExclusive(stagedPath, serializedEvent); if (!staged) { - // eventId is a freshly generated ULID; its staging path can - // only be occupied by a previous crashed attempt of this very - // event, which never promoted. Surface the same conflict shape - // as a visible-path collision. + // For a ULID-numbered run the eventId is freshly generated, so + // its staging path can only be occupied by a previous crashed + // attempt of this very event, which never promoted. A + // slot-numbered run can also collide here with another instance + // that allocated the same slot from its own book. Either way the + // event is not reader-visible, so there is no delta to hand back + // and nothing for the caller to merge: surface the same conflict + // shape as a visible-path collision, and let the allocator + // re-probe on the retry. throw new EntityConflictError( `Event "${eventId}" already exists for run "${effectiveRunId}"` ); @@ -2364,6 +2591,22 @@ export function createEventsStorage( tag ); } + if (slotMode) { + // Losing a slot means someone else's event occupies this position, + // so the log this event was derived from is missing at least that + // event — the whole proposed event is stale, not just its id. Hand + // back what the caller is missing so it can merge, replay and + // re-propose, and forget the run's book so the next allocation + // re-reads the log this instance evidently does not have. + // + // Reaching here means the slot was taken *after* the pre-check at + // the claim site, so the entity this event was going to describe + // has already been materialized. Only two storage instances + // sharing a directory can do that, since one instance's book + // hands the same slot to nobody else. + slots.forget(effectiveRunId); + throw await slotConflict(effectiveRunId, eventId, params); + } throw new EntityConflictError( `Event "${eventId}" already exists for run "${effectiveRunId}"` ); @@ -2372,6 +2615,7 @@ export function createEventsStorage( // The event is now committed; cache it so an immediate sequential // replay can serve it without rereading from disk. rememberStoredEvent(event, eventPath, serializedEvent); + slots.observe(effectiveRunId, eventId); // Write the hook entity ONLY now that the event publish has // committed. Doing this earlier (in the `hook_created` diff --git a/packages/world-local/src/storage/helpers.ts b/packages/world-local/src/storage/helpers.ts index d7c2cc78bb..2817fb0574 100644 --- a/packages/world-local/src/storage/helpers.ts +++ b/packages/world-local/src/storage/helpers.ts @@ -1,6 +1,7 @@ import { createHash } from 'node:crypto'; import fs from 'node:fs/promises'; import path from 'node:path'; +import { FIRST_SLOT, maxSlotOf, slotEventId } from '@workflow/world'; import { decodeTime, monotonicFactory } from 'ulid'; import { hasTag, @@ -208,6 +209,46 @@ export async function reapPendingHookEvents( } } +/** + * The event ids of `runId` that are visible in the given tag's view, read from + * the event filenames alone — no file contents, so the cost is one `readdir` + * however large the log is. + * + * A missing `events` directory means the run provably has no events yet. Any + * other failure is thrown: callers derive an event key from this scan, and a + * silently short answer would mint a key that collides with, or fails to + * dominate, an event that is actually there. + */ +export async function listRunEventIds( + basedir: string, + runId: string, + tag?: string +): Promise { + let files: string[] = []; + try { + files = await fs.readdir(path.join(basedir, 'events')); + } catch (error) { + if ((error as NodeJS.ErrnoException).code !== 'ENOENT') { + throw error; + } + } + const prefix = `${runId}-`; + const eventIds: string[] = []; + for (const file of files) { + if (!file.startsWith(prefix) || !file.endsWith('.json')) { + continue; + } + const fileId = file.slice(0, -'.json'.length); + // Mirror read visibility: untagged files are visible to every tag, + // tagged files only to their own tag. + if (!isUntagged(fileId) && !(tag && hasTag(fileId, tag))) { + continue; + } + eventIds.push(stripTag(fileId).slice(prefix.length)); + } + return eventIds; +} + /** * Mint an event key (eventId + createdAt) that sorts strictly AFTER every * reader-visible event of the run in the given tag's view. @@ -229,38 +270,35 @@ export async function reapPendingHookEvents( * >= every visible event's `createdAt`, which was stamped at that event's * `createImpl()` entry — before its publish, and thus before this call. * Equal-`createdAt` ties fall to the strictly-dominant eventId. + * + * A slot-numbered run takes the slot above the highest visible one, which + * dominates by construction, paired with the wall clock — `createdAt` needs + * only to be >= every visible one, by the same argument as above. This is + * the one allocation that deliberately does *not* fill a hole below the max: + * a lower slot would sort before the events it has to follow, and density + * matters less here than replay order, since a hole below a terminal event + * means the run already lost an event it can never write. */ export async function mintRunDominantEventKey( basedir: string, runId: string, - tag?: string + tag: string | undefined, + slotMode: boolean ): Promise<{ eventId: string; createdAt: Date }> { - let files: string[] = []; - try { - files = await fs.readdir(path.join(basedir, 'events')); - } catch (error) { - // Only ENOENT ("no events directory yet") means there is provably - // nothing visible to dominate. Any other failure would silently mint a - // wall-clock key with no dominance guarantee over an already-accepted - // hook — abort the terminal transition instead; its retry re-runs this - // scan. - if ((error as NodeJS.ErrnoException).code !== 'ENOENT') { - throw error; - } + const eventIds = await listRunEventIds(basedir, runId, tag); + if (slotMode) { + // Above every event on disk, and above the run's own first slot even when + // that event has not landed yet: only `run_created` may occupy it, and a + // terminal event is never the run's first. + return { + eventId: slotEventId( + Math.max(maxSlotOf(eventIds.map(toEventRef)), FIRST_SLOT) + 1 + ), + createdAt: new Date(), + }; } - const prefix = `${runId}-`; let maxUlid: string | null = null; - for (const file of files) { - if (!file.startsWith(prefix) || !file.endsWith('.json')) { - continue; - } - const fileId = file.slice(0, -'.json'.length); - // Mirror read visibility: untagged files are visible to every tag, - // tagged files only to their own tag. - if (!isUntagged(fileId) && !(tag && hasTag(fileId, tag))) { - continue; - } - const candidate = stripTag(fileId).slice(prefix.length); + for (const candidate of eventIds) { if (!maxUlid || candidate > maxUlid) { maxUlid = candidate; } @@ -279,6 +317,10 @@ export async function mintRunDominantEventKey( return { eventId: `evnt_${monotonicUlid(ts)}`, createdAt: new Date(ts) }; } +function toEventRef(eventId: string): { eventId: string } { + return { eventId }; +} + /** * Path of the exclusive-create claim file that reserves a hook token. */ diff --git a/packages/world-local/src/storage/slot-identity.test.ts b/packages/world-local/src/storage/slot-identity.test.ts new file mode 100644 index 0000000000..00d8b71cce --- /dev/null +++ b/packages/world-local/src/storage/slot-identity.test.ts @@ -0,0 +1,281 @@ +import { promises as fs } from 'node:fs'; +import os from 'node:os'; +import path from 'node:path'; +import { SlotConflictError } from '@workflow/errors'; +import type { Storage } from '@workflow/world'; +import { + FIRST_SLOT, + maxSlotOf, + SPEC_VERSION_CURRENT, + SPEC_VERSION_SLOT_IDENTITY, + slotEventId, + slotFromId, +} from '@workflow/world'; +import { afterEach, beforeEach, describe, expect, it } from 'vitest'; +import { createStorage } from './index.js'; + +let testDir: string; +let storage: Storage; + +beforeEach(async () => { + testDir = await fs.mkdtemp(path.join(os.tmpdir(), 'slot-identity-')); + storage = createStorage(testDir); +}); + +afterEach(async () => { + await fs.rm(testDir, { recursive: true, force: true }); +}); + +/** Start a run whose events are numbered by slot, and return its id. */ +async function newSlotRun(): Promise { + const result = await storage.events.create(null, { + eventType: 'run_created', + specVersion: SPEC_VERSION_SLOT_IDENTITY, + eventData: { + deploymentId: 'dpl_test', + workflowName: 'test-workflow', + input: new Uint8Array(), + }, + }); + if (!result.run) { + throw new Error('Expected run to be created'); + } + return result.run.runId; +} + +/** + * The slots of the run's log, in list order. The page size is explicit: the + * default would silently truncate a fan-out and make a dense log look sparse. + */ +async function slotsOf(runId: string): Promise { + const { data } = await eventsOf(runId); + return data.map((event) => slotFromId(event.eventId) ?? -1); +} + +function eventsOf(runId: string) { + return storage.events.list({ runId, pagination: { limit: 500 } }); +} + +async function createStep( + runId: string, + stepId: string, + eventId?: string +): Promise { + const result = await storage.events.create( + runId, + { + eventType: 'step_created', + specVersion: SPEC_VERSION_SLOT_IDENTITY, + correlationId: stepId, + eventData: { stepName: 'a-step', input: new Uint8Array() }, + }, + eventId === undefined ? undefined : { eventId } + ); + if (!result.event) { + throw new Error('Expected an event'); + } + return result.event.eventId; +} + +describe('numbering', () => { + it('puts run_created in the first slot', async () => { + const runId = await newSlotRun(); + await expect(slotsOf(runId)).resolves.toEqual([FIRST_SLOT]); + }); + + it('allocates dense slots for writers that hold no log', async () => { + // A step completion reporting in, a cancellation from an API call: the + // caller has no event log, so the world numbers the event for it. + const runId = await newSlotRun(); + await createStep(runId, 'step_a'); + await createStep(runId, 'step_b'); + await expect(slotsOf(runId)).resolves.toEqual([1, 2, 3]); + }); + + it('honours a slot the caller claims', async () => { + const runId = await newSlotRun(); + const eventId = await createStep(runId, 'step_a', slotEventId(2)); + expect(eventId).toBe(slotEventId(2)); + await expect(slotsOf(runId)).resolves.toEqual([1, 2]); + }); + + it('keeps a burst of concurrent writers dense', async () => { + // The suspension flush issues every op at once. Density is what lets a + // reader prove its log is complete, so a burst must not leave holes. + const runId = await newSlotRun(); + const ids = await Promise.all( + Array.from({ length: 20 }, (_, index) => + createStep(runId, `step_${index}`) + ) + ); + expect(new Set(ids).size).toBe(ids.length); + const slots = await slotsOf(runId); + expect([...slots].sort((a, b) => a - b)).toEqual( + Array.from({ length: ids.length + 1 }, (_, index) => FIRST_SLOT + index) + ); + }); + + it('proves completeness: the highest slot is the event count', async () => { + const runId = await newSlotRun(); + await Promise.all( + Array.from({ length: 5 }, (_, index) => + createStep(runId, `step_${index}`) + ) + ); + const { data } = await eventsOf(runId); + expect(maxSlotOf(data)).toBe(data.length); + }); + + it('leaves no hole behind a rejected write', async () => { + // The rejected op's slot sits below its concurrent sibling's, and a hole + // below a published event can never be filled. + const runId = await newSlotRun(); + const [rejected, accepted] = await Promise.allSettled([ + storage.events.create(runId, { + eventType: 'step_completed', + specVersion: SPEC_VERSION_SLOT_IDENTITY, + correlationId: 'step_never_created', + eventData: { output: new Uint8Array() }, + }), + createStep(runId, 'step_a'), + ]); + expect(rejected.status).toBe('rejected'); + expect(accepted.status).toBe('fulfilled'); + await createStep(runId, 'step_b'); + const slots = await slotsOf(runId); + expect([...slots].sort((a, b) => a - b)).toEqual([1, 2, 3]); + }); +}); + +describe('mode is pinned to the run', () => { + it('rejects a slot id claimed on a ULID-numbered run', async () => { + const created = await storage.events.create(null, { + eventType: 'run_created', + specVersion: SPEC_VERSION_CURRENT, + eventData: { + deploymentId: 'dpl_test', + workflowName: 'test-workflow', + input: new Uint8Array(), + }, + }); + const runId = created.run?.runId as string; + await expect(createStep(runId, 'step_a', slotEventId(2))).rejects.toThrow( + /not numbered by slot/ + ); + }); + + it('rejects a ULID id claimed on a slot-numbered run', async () => { + const runId = await newSlotRun(); + await expect( + createStep(runId, 'step_a', 'evnt_01K5Z0000000000000000000AA') + ).rejects.toThrow(/not a slot id/); + }); + + it('ignores the spec version of later requests', async () => { + // A run is in exactly one mode for life; only what was persisted decides. + const runId = await newSlotRun(); + const result = await storage.events.create(runId, { + eventType: 'step_created', + specVersion: SPEC_VERSION_CURRENT, + correlationId: 'step_a', + eventData: { stepName: 'a-step', input: new Uint8Array() }, + }); + expect(slotFromId(result.event?.eventId ?? '')).toBe(2); + }); +}); + +describe('conflict', () => { + it('reports the events the loser is missing', async () => { + const runId = await newSlotRun(); + // Out of band: something else takes the slot this caller was about to + // claim, so the caller's log is provably missing an event. + await createStep(runId, 'step_out_of_band'); + + const conflict = await storage.events + .create( + runId, + { + eventType: 'step_created', + specVersion: SPEC_VERSION_SLOT_IDENTITY, + correlationId: 'step_a', + eventData: { stepName: 'a-step', input: new Uint8Array() }, + }, + { eventId: slotEventId(2), maxSlot: 1 } + ) + .catch((error: unknown) => error); + + expect(SlotConflictError.is(conflict)).toBe(true); + const slotConflict = conflict as SlotConflictError; + expect(slotConflict.status).toBe(409); + expect(slotConflict.eventId).toBe(slotEventId(2)); + expect(slotConflict.events?.map((event) => event.eventId)).toEqual([ + slotEventId(2), + ]); + }); + + it('lets the loser re-propose at the next free slot', async () => { + const runId = await newSlotRun(); + await createStep(runId, 'step_out_of_band'); + await expect(createStep(runId, 'step_a', slotEventId(2))).rejects.toThrow( + SlotConflictError + ); + // Merging the delta moves the caller's own numbering forward by one. + const eventId = await createStep(runId, 'step_a', slotEventId(3)); + expect(eventId).toBe(slotEventId(3)); + await expect(slotsOf(runId)).resolves.toEqual([1, 2, 3]); + }); + + it('excludes events the loser already holds from the delta', async () => { + const runId = await newSlotRun(); + await createStep(runId, 'step_one'); + await createStep(runId, 'step_two'); + + const conflict = await storage.events + .create( + runId, + { + eventType: 'step_created', + specVersion: SPEC_VERSION_SLOT_IDENTITY, + correlationId: 'step_a', + eventData: { stepName: 'a-step', input: new Uint8Array() }, + }, + { eventId: slotEventId(2), maxSlot: 2 } + ) + .catch((error: unknown) => error); + + // Slots 1 and 2 are at or below what the caller had; only 3 is news. + expect( + (conflict as SlotConflictError).events?.map((event) => event.eventId) + ).toEqual([slotEventId(3)]); + }); + + it('conflicts across two storage instances sharing a directory', async () => { + // Two instances keep independent books, so the exclusive write — not the + // book — is what decides who owns a slot. + const runId = await newSlotRun(); + const other = createStorage(testDir); + const [first, second] = await Promise.allSettled([ + storage.events.create(runId, { + eventType: 'step_created', + specVersion: SPEC_VERSION_SLOT_IDENTITY, + correlationId: 'step_a', + eventData: { stepName: 'a-step', input: new Uint8Array() }, + }), + other.events.create(runId, { + eventType: 'step_created', + specVersion: SPEC_VERSION_SLOT_IDENTITY, + correlationId: 'step_b', + eventData: { stepName: 'b-step', input: new Uint8Array() }, + }), + ]); + const outcomes = [first, second]; + expect(outcomes.filter((o) => o.status === 'fulfilled')).toHaveLength(1); + const rejection = outcomes.find((o) => o.status === 'rejected'); + expect( + SlotConflictError.is( + (rejection as PromiseRejectedResult | undefined)?.reason + ) + ).toBe(true); + await expect(slotsOf(runId)).resolves.toEqual([1, 2]); + }); +}); diff --git a/packages/world-local/src/storage/slots.test.ts b/packages/world-local/src/storage/slots.test.ts new file mode 100644 index 0000000000..42c962f128 --- /dev/null +++ b/packages/world-local/src/storage/slots.test.ts @@ -0,0 +1,241 @@ +import fs from 'node:fs/promises'; +import os from 'node:os'; +import path from 'node:path'; +import { + FIRST_SLOT, + SPEC_VERSION_CURRENT, + SPEC_VERSION_SLOT_IDENTITY, + slotEventId, +} from '@workflow/world'; +import { afterEach, beforeEach, describe, expect, it } from 'vitest'; +import { createSlotBook, RUN_CREATED_SLOT } from './slots.js'; + +let basedir: string; + +beforeEach(async () => { + basedir = await fs.mkdtemp(path.join(os.tmpdir(), 'slot-book-')); +}); + +afterEach(async () => { + await fs.rm(basedir, { recursive: true, force: true }); +}); + +const RUN_ID = 'wrun_01K0000000000000000000TEST'; + +async function writeRun(specVersion: number): Promise { + await fs.mkdir(path.join(basedir, 'runs'), { recursive: true }); + await fs.writeFile( + path.join(basedir, 'runs', `${RUN_ID}.json`), + JSON.stringify({ + runId: RUN_ID, + deploymentId: 'dpl_test', + status: 'running', + workflowName: 'test', + specVersion, + input: [], + attributes: {}, + createdAt: new Date().toISOString(), + updatedAt: new Date().toISOString(), + }) + ); +} + +async function writeEvents(...slots: number[]): Promise { + await fs.mkdir(path.join(basedir, 'events'), { recursive: true }); + for (const slot of slots) { + await fs.writeFile( + path.join(basedir, 'events', `${RUN_ID}-${slotEventId(slot)}.json`), + '{}' + ); + } +} + +describe('usesSlots', () => { + it('reads the mode off the persisted run, not the build', async () => { + await writeRun(SPEC_VERSION_SLOT_IDENTITY); + await expect(createSlotBook(basedir).usesSlots(RUN_ID)).resolves.toBe(true); + + await writeRun(SPEC_VERSION_CURRENT); + await expect(createSlotBook(basedir).usesSlots(RUN_ID)).resolves.toBe( + false + ); + }); + + it('re-reads until the run exists', async () => { + // The resilient-start path writes run_started before the run entity, so a + // cached "no" taken from the missing run would strand a slot-numbered run + // on ULID ids for the rest of the process's life. + const book = createSlotBook(basedir); + await expect(book.usesSlots(RUN_ID)).resolves.toBe(false); + await writeRun(SPEC_VERSION_SLOT_IDENTITY); + await expect(book.usesSlots(RUN_ID)).resolves.toBe(true); + }); + + it('prefers its own tagged run over the untagged one', async () => { + await writeRun(SPEC_VERSION_SLOT_IDENTITY); + await fs.rename( + path.join(basedir, 'runs', `${RUN_ID}.json`), + path.join(basedir, 'runs', `${RUN_ID}.mine.json`) + ); + await writeRun(SPEC_VERSION_CURRENT); + + await expect( + createSlotBook(basedir, 'mine').usesSlots(RUN_ID) + ).resolves.toBe(true); + await expect( + createSlotBook(basedir, 'other').usesSlots(RUN_ID) + ).resolves.toBe(false); + }); +}); + +describe('reserve', () => { + it('starts at the first slot for a run with no events', async () => { + await expect(createSlotBook(basedir).reserve(RUN_ID)).resolves.toBe( + FIRST_SLOT + ); + }); + + it('continues above the highest slot already on disk', async () => { + await writeEvents(1, 2, 3); + await expect(createSlotBook(basedir).reserve(RUN_ID)).resolves.toBe(4); + }); + + it('fills a hole left in the persisted log', async () => { + // Density is the whole point of the scheme, so a gap that somehow exists is + // reclaimed rather than skipped over forever. + await writeEvents(1, 3); + const book = createSlotBook(basedir); + await expect(book.reserve(RUN_ID)).resolves.toBe(2); + await expect(book.reserve(RUN_ID)).resolves.toBe(4); + }); + + it('hands a synchronous burst distinct consecutive slots', async () => { + // The suspension flush issues every op concurrently; with a plain + // "max + 1" they would all pick the same slot and all but one would fail. + const book = createSlotBook(basedir); + const slots = await Promise.all( + Array.from({ length: 20 }, () => book.reserve(RUN_ID)) + ); + expect([...slots].sort((a, b) => a - b)).toEqual( + Array.from({ length: 20 }, (_, index) => FIRST_SLOT + index) + ); + }); + + it('shares one disk scan across concurrent first callers', async () => { + await writeEvents(1); + const book = createSlotBook(basedir); + const slots = await Promise.all([ + book.reserve(RUN_ID), + book.reserve(RUN_ID), + ]); + expect([...slots].sort((a, b) => a - b)).toEqual([2, 3]); + }); + + it('honours a floor, so a concurrent event cannot take run_created’s slot', async () => { + // start() publishes the run entity before its `run_created` event and + // issues the queue send in parallel, so the delivery's `run_started` can + // allocate while slot 1 is still in flight. + const book = createSlotBook(basedir); + await expect(book.reserve(RUN_ID, RUN_CREATED_SLOT + 1)).resolves.toBe( + RUN_CREATED_SLOT + 1 + ); + }); + + it('leaves slots below the floor allocatable', async () => { + // A floored search skips that range without looking at it, so it proves + // nothing about it — `run_created` must still find its own slot free. + const book = createSlotBook(basedir); + await expect(book.reserve(RUN_ID, RUN_CREATED_SLOT + 1)).resolves.toBe( + RUN_CREATED_SLOT + 1 + ); + await expect(book.reserve(RUN_ID, RUN_CREATED_SLOT)).resolves.toBe( + RUN_CREATED_SLOT + ); + }); + + it('keeps runs independent', async () => { + await writeEvents(1, 2); + const book = createSlotBook(basedir); + await expect(book.reserve(RUN_ID)).resolves.toBe(3); + await expect(book.reserve('wrun_01K0000000000000000000OTHR')).resolves.toBe( + FIRST_SLOT + ); + }); +}); + +describe('release', () => { + it('gives an abandoned interior slot to the next caller', async () => { + // A rejected op must not strand the slot below its concurrent siblings': + // that hole can never be filled once a later slot is published. + const book = createSlotBook(basedir); + const [first, second] = await Promise.all([ + book.reserve(RUN_ID), + book.reserve(RUN_ID), + ]); + book.release(RUN_ID, first); + await expect(book.reserve(RUN_ID)).resolves.toBe(first); + await expect(book.reserve(RUN_ID)).resolves.toBe(second + 1); + }); + + it('does not resurrect a slot that was published', async () => { + const book = createSlotBook(basedir); + const slot = await book.reserve(RUN_ID); + book.observe(RUN_ID, slotEventId(slot)); + book.release(RUN_ID, slot); + await expect(book.reserve(RUN_ID)).resolves.toBe(slot + 1); + }); +}); + +describe('isWritten', () => { + it('reads the log to answer for a run it has never seen', async () => { + await writeEvents(1, 2); + const book = createSlotBook(basedir); + await expect(book.isWritten(RUN_ID, 2)).resolves.toBe(true); + await expect(book.isWritten(RUN_ID, 3)).resolves.toBe(false); + }); + + it('is false for a slot that is only reserved', async () => { + // A reservation is not a publish, so a caller claiming the slot has to be + // allowed through to the write that actually decides it. + const book = createSlotBook(basedir); + const slot = await book.reserve(RUN_ID); + await expect(book.isWritten(RUN_ID, slot)).resolves.toBe(false); + }); +}); + +describe('observe', () => { + it('never hands out a slot claimed by the client', async () => { + const book = createSlotBook(basedir); + await book.reserve(RUN_ID); + book.observe(RUN_ID, slotEventId(5)); + const next = await book.reserve(RUN_ID); + expect(next).not.toBe(5); + expect(next).toBe(2); + }); + + it('ignores ULID event ids', async () => { + const book = createSlotBook(basedir); + await book.reserve(RUN_ID); + book.observe(RUN_ID, 'evnt_01K5Z0000000000000000000AA'); + await expect(book.reserve(RUN_ID)).resolves.toBe(2); + }); +}); + +describe('forget', () => { + it("re-reads the log, picking up another writer's events", async () => { + const book = createSlotBook(basedir); + await expect(book.reserve(RUN_ID)).resolves.toBe(FIRST_SLOT); + await writeEvents(1, 2, 3); + book.forget(RUN_ID); + await expect(book.reserve(RUN_ID)).resolves.toBe(4); + }); + + it('clear() forgets every run', async () => { + await writeEvents(1); + const book = createSlotBook(basedir); + await expect(book.reserve(RUN_ID)).resolves.toBe(2); + await writeEvents(2, 3); + book.clear(); + await expect(book.reserve(RUN_ID)).resolves.toBe(4); + }); +}); diff --git a/packages/world-local/src/storage/slots.ts b/packages/world-local/src/storage/slots.ts new file mode 100644 index 0000000000..c6b2f017f6 --- /dev/null +++ b/packages/world-local/src/storage/slots.ts @@ -0,0 +1,247 @@ +/** + * Slot allocation for the Local World. + * + * A slot-numbered run names its events by position: `evnt_…001` is the first + * event of the run, `evnt_…002` the second, with no gaps. Density is the point + * — it is what lets a reader prove its loaded log is complete — so an allocator + * must never leave a slot permanently unwritten. + * + * Three properties do the work: + * + * - Handing out a slot is a *synchronous* set operation, so concurrent + * callers in one process get distinct slots with no lock. The only await is + * seeding from disk, which is memoized per run. + * - An allocation always picks the lowest slot that is neither written nor + * outstanding, so a reservation that is abandoned (its create threw a + * validation error) is handed to the next caller instead of leaving an + * interior hole. This matters under fan-out: N concurrent step_completed + * writes reserve N consecutive slots, and one rejected op must not strand + * the slot below its siblings'. + * - The event publish is `writeExclusive`, which is the authority. The book is + * a hint: when it turns out to be stale (another process wrote the slot), + * the publish fails and the caller is told so, rather than a duplicate being + * written or a slot being skipped. + * + * The book is per storage instance, and two instances may share a data + * directory (the cross-process convergence tests rely on exactly that). Their + * books are then independent, and the loser of a collision gets a conflict it + * has to resolve by reloading — the same contract as the networked worlds. + */ + +import type { WorkflowRun } from '@workflow/world'; +import { + FIRST_SLOT, + slotFromId, + usesSlotIdentity, + WorkflowRunSchema, +} from '@workflow/world'; +import { readJSONWithFallback } from '../fs.js'; +import { listRunEventIds } from './helpers.js'; + +interface RunSlots { + /** Slots proven to be on disk. */ + written: Set; + /** Slots handed out whose publish has not resolved yet. */ + outstanding: Set; + /** Lowest slot that might still be free; never decreases except on release. */ + searchFrom: number; +} + +export interface SlotBook { + /** + * Whether `runId`'s events are numbered by slot, read from the run's + * persisted `specVersion` — never from the build, so a run stays in the mode + * it was created in for life. A run that does not exist yet is not + * slot-numbered, and that answer is not cached: the resilient-start path + * creates the run moments later, and caching "no" would strand it on ULIDs + * for the rest of this process's life. + */ + usesSlots(runId: string): Promise; + /** + * Reserves the lowest free slot of `runId`, at or above `minSlot`. Distinct + * for every concurrent caller; the publish still has to prove the slot was + * actually free. + * + * `minSlot` is how the run's first slot is kept for its own `run_created`: + * that event's slot needs no allocation, and it may not be on disk yet when a + * concurrent `run_started` allocates (start() issues the creation and the + * queue send in parallel, and the run entity is published before its event). + * Slots below `minSlot` stay allocatable for a later caller, so holding one + * back leaves the log dense. + */ + reserve(runId: string, minSlot?: number): Promise; + /** + * Records that a caller claimed `slot` itself, so an allocation running + * alongside it picks a different one. Reserved and released on the same terms + * as {@link reserve}: the claim is only a hint until the publish proves it. + */ + claim(runId: string, slot: number): void; + /** + * Whether `slot` is already occupied by a published event, seeding from disk + * if this run has not been read yet. + * + * Lets a doomed claim be rejected *before* the create materializes its step, + * hook or wait: the entity mutation runs ahead of the event publish, so a + * claim that only fails at the publish leaves an entity behind with no event, + * and the caller's re-proposal at the next slot then collides with its own + * orphan. A `false` here is not a promise — the publish is still the + * authority — but it turns the case that actually happens (a caller numbering + * from a stale log) into a clean conflict. + */ + isWritten(runId: string, slot: number): Promise; + /** + * Returns a reserved or claimed slot that was never published, so the next + * caller takes it instead of it becoming a hole. + */ + release(runId: string, slot: number): void; + /** Records a published event id, so it is never handed out again. */ + observe(runId: string, eventId: string): void; + /** Drops what is cached for `runId`, so the next reservation re-reads disk. */ + forget(runId: string): void; + /** Drops everything cached (the data directory was cleared out from under us). */ + clear(): void; +} + +export function createSlotBook(basedir: string, tag?: string): SlotBook { + /** runId → whether the run is slot-numbered, memoized once it exists. */ + const modes = new Map(); + const books = new Map(); + /** runId → in-flight seed scan, so concurrent first callers share one scan. */ + const seeds = new Map>(); + + async function readMode(runId: string): Promise { + const run = await readJSONWithFallback( + basedir, + 'runs', + runId, + WorkflowRunSchema, + tag + ); + return run ? usesSlotIdentity(run.specVersion) : false; + } + + async function seed(runId: string): Promise { + const eventIds = await listRunEventIds(basedir, runId, tag); + const written = new Set(); + for (const eventId of eventIds) { + const slot = slotFromId(eventId); + if (slot !== undefined) { + written.add(slot); + } + } + const book: RunSlots = { + written, + outstanding: new Set(), + searchFrom: FIRST_SLOT, + }; + books.set(runId, book); + return book; + } + + /** The run's book, seeding it from disk once for all concurrent callers. */ + function open(runId: string): RunSlots | Promise { + const known = books.get(runId); + if (known) { + return known; + } + let pending = seeds.get(runId); + if (!pending) { + pending = seed(runId).finally(() => seeds.delete(runId)); + seeds.set(runId, pending); + } + return pending; + } + + function take(book: RunSlots, minSlot: number): number { + let slot = Math.max(book.searchFrom, minSlot); + while (book.written.has(slot) || book.outstanding.has(slot)) { + slot += 1; + } + book.outstanding.add(slot); + if (minSlot <= book.searchFrom) { + // Only an unfloored search proves everything below the slot it landed on + // is taken. A floored one skipped that range without looking, and those + // slots are still free for a caller that may take them. + book.searchFrom = slot; + } + return slot; + } + + return { + async usesSlots(runId) { + const cached = modes.get(runId); + if (cached !== undefined) { + return cached; + } + const mode = await readMode(runId); + // `false` here can mean "run not created yet" as well as "ULID run", and + // only the run's own absence is transient — so remember the positive + // answer eagerly and re-read until the run exists. + if (mode) { + modes.set(runId, true); + } + return mode; + }, + + async reserve(runId, minSlot = FIRST_SLOT) { + const opened = open(runId); + // Awaiting a book that is already in hand would yield to the microtask + // queue and let a concurrent caller take the same slot. + return take(opened instanceof Promise ? await opened : opened, minSlot); + }, + + claim(runId, slot) { + // No book means nothing is allocating for this run in this instance yet, + // and the seed scan that starts one reads the claim off disk if it landed. + books.get(runId)?.outstanding.add(slot); + }, + + async isWritten(runId, slot) { + const book = await open(runId); + return book.written.has(slot); + }, + + release(runId, slot) { + const book = books.get(runId); + if (!book) { + return; + } + book.outstanding.delete(slot); + if (slot < book.searchFrom) { + book.searchFrom = slot; + } + }, + + observe(runId, eventId) { + const book = books.get(runId); + if (!book) { + // Nothing to keep consistent: the slot is on disk by the time this is + // called, so the eventual seed scan picks it up. + return; + } + const slot = slotFromId(eventId); + if (slot === undefined) { + return; + } + book.written.add(slot); + book.outstanding.delete(slot); + }, + + forget(runId) { + modes.delete(runId); + books.delete(runId); + }, + + clear() { + modes.clear(); + books.clear(); + }, + }; +} + +/** + * The slot a run's first event occupies. A run's own `run_created` is the only + * event that can be numbered without consulting the log, because there is + * provably nothing before it. + */ +export const RUN_CREATED_SLOT = FIRST_SLOT; diff --git a/packages/world-postgres/src/drizzle/migrations/0018_run_scoped_event_and_step_keys.sql b/packages/world-postgres/src/drizzle/migrations/0018_run_scoped_event_and_step_keys.sql new file mode 100644 index 0000000000..0764dee02d --- /dev/null +++ b/packages/world-postgres/src/drizzle/migrations/0018_run_scoped_event_and_step_keys.sql @@ -0,0 +1,13 @@ +-- Event ids and step ids are unique per run, not globally. Under slot identity +-- (spec 6) every run numbers its own log from 1, so "evnt_0...001" and +-- "step_0...001" exist once per run and the old global primary keys would make +-- the second run to reach slot 1 collide with the first. +-- +-- The run leads both keys so the existing run-scoped range scans stay a single +-- index seek; that also makes the standalone run_id indexes redundant. +ALTER TABLE "workflow"."workflow_events" DROP CONSTRAINT IF EXISTS "workflow_events_pkey";--> statement-breakpoint +ALTER TABLE "workflow"."workflow_events" ADD CONSTRAINT "workflow_events_run_id_id_pk" PRIMARY KEY("run_id","id");--> statement-breakpoint +DROP INDEX IF EXISTS "workflow"."workflow_events_run_id_index";--> statement-breakpoint +ALTER TABLE "workflow"."workflow_steps" DROP CONSTRAINT IF EXISTS "workflow_steps_pkey";--> statement-breakpoint +ALTER TABLE "workflow"."workflow_steps" ADD CONSTRAINT "workflow_steps_run_id_step_id_pk" PRIMARY KEY("run_id","step_id");--> statement-breakpoint +DROP INDEX IF EXISTS "workflow"."workflow_steps_run_id_index"; diff --git a/packages/world-postgres/src/drizzle/migrations/meta/_journal.json b/packages/world-postgres/src/drizzle/migrations/meta/_journal.json index b7fb5d8215..9dca967a5b 100644 --- a/packages/world-postgres/src/drizzle/migrations/meta/_journal.json +++ b/packages/world-postgres/src/drizzle/migrations/meta/_journal.json @@ -127,6 +127,13 @@ "when": 1785283200000, "tag": "0017_add_hook_resume_context", "breakpoints": true + }, + { + "idx": 18, + "version": "7", + "when": 1785801600000, + "tag": "0018_run_scoped_event_and_step_keys", + "breakpoints": true } ] } diff --git a/packages/world-postgres/src/drizzle/schema.ts b/packages/world-postgres/src/drizzle/schema.ts index 6ffb21abcb..df58f73eb8 100644 --- a/packages/world-postgres/src/drizzle/schema.ts +++ b/packages/world-postgres/src/drizzle/schema.ts @@ -133,7 +133,7 @@ export const runs = schema.table( export const events = schema.table( 'workflow_events', { - eventId: varchar('id').primaryKey(), + eventId: varchar('id').notNull(), eventType: varchar('type').$type().notNull(), correlationId: varchar('correlation_id'), createdAt: timestamp('created_at').defaultNow().notNull(), @@ -146,7 +146,11 @@ export const events = schema.table( Cborized & { eventData?: undefined }, 'eventData'> >, (tb) => [ - index().on(tb.runId), + // Event ids are only unique within their run: under slot identity every run + // numbers its own log from 1, so `evnt_0…001` exists once per run. The run + // leads the key so the range scans in `list` stay a single index seek, and + // it subsumes the plain `run_id` index the table used to carry. + primaryKey({ columns: [tb.runId, tb.eventId] }), index().on(tb.correlationId), // Runtime-correlated one-shot events must be unique per (run, correlation) // — without @@ -167,7 +171,7 @@ export const steps = schema.table( 'workflow_steps', { runId: varchar('run_id').notNull(), - stepId: varchar('step_id').primaryKey(), + stepId: varchar('step_id').notNull(), stepName: varchar('step_name').notNull(), status: stepStatus('status').notNull(), /** @deprecated */ @@ -203,7 +207,13 @@ export const steps = schema.table( 'output' | 'input' | 'error' > >, - (tb) => [index().on(tb.runId), index().on(tb.status)] + (tb) => [ + // A step id is a correlation id, which under slot identity is only unique + // within its run — same reasoning as `workflow_events`. Every step query in + // this world is already run-scoped, so the run leads the key. + primaryKey({ columns: [tb.runId, tb.stepId] }), + index().on(tb.status), + ] ); export const hooks = schema.table( diff --git a/packages/world-postgres/src/index.ts b/packages/world-postgres/src/index.ts index 430cea0812..0db47b89e2 100644 --- a/packages/world-postgres/src/index.ts +++ b/packages/world-postgres/src/index.ts @@ -1,5 +1,5 @@ import type { Storage, World } from '@workflow/world'; -import { reenqueueActiveRuns, SPEC_VERSION_CURRENT } from '@workflow/world'; +import { mintedSpecVersion, reenqueueActiveRuns } from '@workflow/world'; import { Pool } from 'pg'; import type { PostgresWorldConfig } from './config.js'; import { createClient, type Drizzle } from './drizzle/index.js'; @@ -63,7 +63,10 @@ export function createWorld( const streamer = createStreamer(pool, drizzle); return { - specVersion: SPEC_VERSION_CURRENT, + // What this world stamps on new runs, which is not the newest version it + // can read: slot identity is readable everywhere and minted only where + // WORKFLOW_SLOT_IDENTITY is set. + specVersion: mintedSpecVersion(), ...storage, ...streamer, ...queue, diff --git a/packages/world-postgres/src/slots.ts b/packages/world-postgres/src/slots.ts new file mode 100644 index 0000000000..89435810a9 --- /dev/null +++ b/packages/world-postgres/src/slots.ts @@ -0,0 +1,225 @@ +/** + * Slot identity for the postgres world. + * + * A slot-numbered run names its events by position: `evnt_…001` is the first + * event of the run, `evnt_…002` the second, with no gaps. Density is the point + * — it is what lets a reader prove its copy of a log is complete — so the two + * things this module has to get right are that a position is written at most + * once and that a lost position is never left behind as a hole. + * + * The authority for both is the events table's primary key, `(run_id, id)`: the + * INSERT either lands or raises a unique violation, and a writer that loses the + * race is retried at a position that is still free rather than abandoning the + * one it lost. The probe below is only ever a hint about where to try next. + */ + +import { WorkflowWorldError } from '@workflow/errors'; +import { FIRST_SLOT, slotEventId, slotFromId } from '@workflow/world'; +import { and, desc, eq } from 'drizzle-orm'; +import { type Drizzle, Schema } from './drizzle/index.js'; + +/** + * The slot a run's own `run_created` occupies. Nothing in a run precedes its + * creation, so this one position needs no allocation, and every other event of + * the run searches above it — including the event that happens to reach storage + * first, which on the start path is routinely `run_started`. + */ +export const RUN_CREATED_SLOT = FIRST_SLOT; + +/** First backoff after losing a position; doubled each round. */ +export const SLOT_RETRY_BASE_MS = 5; + +/** Ceiling for a single backoff, so a contended run keeps making attempts. */ +export const SLOT_RETRY_MAX_DELAY_MS = 250; + +/** + * How long a writer keeps looking for a free position before giving up. + * Exhausting it surfaces as a 503, so the caller — in practice a queue + * delivery — retries the whole operation instead of the run stalling on it. + */ +export const SLOT_RETRY_BUDGET_MS = 30_000; + +/** Postgres unique-violation code. */ +const UNIQUE_VIOLATION = '23505'; + +/** + * Whether an error says the position a write aimed at is already occupied. + * + * Drizzle wraps the pg error, so the code can sit on the error or on its cause. + * Both the name drizzle generates for the composite key and the name postgres + * gives an inline `PRIMARY KEY` are accepted, so a database whose key predates + * the run-scoped migration still classifies correctly. + */ +export function isEventKeyViolation(error: unknown): boolean { + const pg = (error as { code?: string; constraint?: string }).code + ? (error as { code?: string; constraint?: string }) + : ((error as { cause?: { code?: string; constraint?: string } }).cause ?? + {}); + return ( + pg.code === UNIQUE_VIOLATION && + (pg.constraint === 'workflow_events_run_id_id_pk' || + pg.constraint === 'workflow_events_pkey') + ); +} + +/** + * The highest event id in a run's log, or undefined when the log is empty. + * + * One backwards scan of the `(run_id, id)` primary key. Ids are fixed-width + * within a scheme, so for a slot-numbered run the highest id names the highest + * written position — and because a log holds ids of exactly one scheme, that id + * also reports which scheme the run was created with. + */ +export async function highestEventId( + drizzle: Drizzle, + runId: string +): Promise { + const [row] = await drizzle + .select({ eventId: Schema.events.eventId }) + .from(Schema.events) + .where(eq(Schema.events.runId, runId)) + .orderBy(desc(Schema.events.eventId)) + .limit(1); + return row?.eventId; +} + +/** The position an id names, or 0 for an empty log or a ULID-numbered one. */ +export function highestSlotOf(eventId: string | undefined): number { + return eventId === undefined ? 0 : (slotFromId(eventId) ?? 0); +} + +/** Whether a run's log already holds `eventId`. */ +export async function eventExists( + drizzle: Drizzle, + runId: string, + eventId: string +): Promise { + const [row] = await drizzle + .select({ eventId: Schema.events.eventId }) + .from(Schema.events) + .where( + and(eq(Schema.events.runId, runId), eq(Schema.events.eventId, eventId)) + ) + .limit(1); + return row !== undefined; +} + +/** Full jitter over an exponentially growing, capped window. */ +export function slotRetryDelay(round: number): number { + return ( + Math.random() * + Math.min(SLOT_RETRY_BASE_MS * 2 ** round, SLOT_RETRY_MAX_DELAY_MS) + ); +} + +/** The event ids a single create publishes. */ +export interface EventIds { + /** + * The id of the event this create returns. Taken on demand: a ULID-numbered + * write mints it inside its transaction, once the row lock that orders it is + * held. + */ + primary: () => string; + /** + * An additional event written in the same breath — the synthetic + * `step_created` of a lazy step start. Its position is allocated here even + * when the caller named its own for the primary event, because a caller that + * defers a `step_created` cannot know it will be synthesized. + */ + extra: () => Promise; +} + +export interface PlaceEventOptions { + /** + * Position the caller named, when it holds the log and claimed one. A claim + * asserts the log is complete up to that position, so losing it is a conflict + * the caller has to resolve rather than something to retry here. + */ + claimedSlot?: number; + /** Lowest position this write may take when allocating. */ + minSlot: number; + /** + * Result of a probe the caller has already made, used for the first attempt + * instead of probing again. Later rounds always re-probe: the log has + * demonstrably moved. + */ + seedHighestEventId?: string | undefined; + /** The conflict raised when a claimed position turns out to be taken. */ + onClaimTaken: () => Promise; + /** Performs the write with the ids it should publish under. */ + write: (ids: EventIds) => Promise; +} + +/** + * Writes an event of a slot-numbered run, at the position the caller claimed or + * at the next free one. + * + * Every round re-probes rather than incrementing a local counter: each round at + * least one writer wins, so re-probing guarantees progress under any amount of + * contention. `write` must leave nothing behind when it raises a unique + * violation — the callers here either write only the event row or wrap their + * materialization in the same transaction, so a lost round rolls back whole. + */ +export async function placeEvent( + drizzle: Drizzle, + runId: string, + options: PlaceEventOptions +): Promise { + const deadline = Date.now() + SLOT_RETRY_BUDGET_MS; + for (let round = 0; ; round++) { + let cursor: number | undefined; + /** + * Positions for this attempt, consecutive from one probe. Deferred so a + * claimed write with no extra event never probes at all. + */ + const take = async (): Promise => { + if (cursor === undefined) { + const highest = + round === 0 && options.seedHighestEventId !== undefined + ? options.seedHighestEventId + : await highestEventId(drizzle, runId); + cursor = Math.max( + highestSlotOf(highest) + 1, + options.minSlot, + // The claimed position is this write's own; an extra event must not + // be handed it. + (options.claimedSlot ?? 0) + 1 + ); + } + return cursor++; + }; + + const primary = + options.claimedSlot === undefined + ? slotEventId(await take()) + : slotEventId(options.claimedSlot); + try { + return await options.write({ + primary: () => primary, + extra: async () => slotEventId(await take()), + }); + } catch (error) { + if (!isEventKeyViolation(error)) { + throw error; + } + // A claimed write can also lose on its extra event's position, which is + // this world's to reallocate — only a claim that is itself taken is the + // caller's problem. + if ( + options.claimedSlot !== undefined && + (await eventExists(drizzle, runId, primary)) + ) { + throw await options.onClaimTaken(); + } + if (Date.now() >= deadline) { + throw new WorkflowWorldError( + `Could not place an event in run "${runId}" within ${SLOT_RETRY_BUDGET_MS}ms of contention`, + { status: 503 } + ); + } + await new Promise((resolve) => + setTimeout(resolve, slotRetryDelay(round)) + ); + } + } +} diff --git a/packages/world-postgres/src/storage.ts b/packages/world-postgres/src/storage.ts index 140597a6a9..878f39758b 100644 --- a/packages/world-postgres/src/storage.ts +++ b/packages/world-postgres/src/storage.ts @@ -3,12 +3,14 @@ import { HookNotFoundError, RunExpiredError, RunNotSupportedError, + SlotConflictError, TooEarlyError, WorkflowRunNotFoundError, WorkflowWorldError, } from '@workflow/errors'; import type { AttributeChange, + CreateEventParams, Event, EventResult, ExperimentalSetAttributesResult, @@ -35,15 +37,20 @@ import { isChildEntityCreationEventType, isHookEventRequiringExistence, isLegacySpecVersion, + isSlotId, isTerminalRunEventType, isTerminalStepStatus, isTerminalWorkflowRunStatus, requiresNewerWorld, SPEC_VERSION_CURRENT, + SPEC_VERSION_MAX_SUPPORTED, StepSchema, + slotEventId, + slotFromId, stripEventDataRefs, TERMINAL_STEP_STATUSES, TERMINAL_WORKFLOW_RUN_STATUSES, + usesSlotIdentity, validateAttributeChanges, validateUlidTimestamp, WorkflowRunSchema, @@ -62,6 +69,13 @@ import { import { monotonicFactory } from 'ulid'; import { type Drizzle, Schema } from './drizzle/index.js'; import type { SerializedContent } from './drizzle/schema.js'; +import { + type EventIds, + eventExists, + highestEventId, + placeEvent, + RUN_CREATED_SLOT, +} from './slots.js'; import { compact } from './util.js'; /** @@ -464,6 +478,64 @@ export function createEventsStorage(drizzle: Drizzle): Storage['events'] { .limit(1) .prepare('events_get_wait_for_validation'); + /** + * The events a caller that just lost a slot is missing: one ascending page of + * the run's log after the cursor it wrote from, minus anything at or below the + * highest slot it already held. Because slots are dense, that second filter is + * exact — a caller cannot be missing an event whose position it can name. + * + * Returned inline with the conflict so the common case (a handful of events + * arrived out of band) costs the caller no extra round-trip. `hasMore` is + * forwarded verbatim: an overflowing delta is the caller's signal to page from + * `cursor` instead of treating this as the whole story. + */ + async function eventsAfterClaim( + runId: string, + params: CreateEventParams | undefined + ): Promise<{ events: Event[]; cursor: string | null; hasMore: boolean }> { + const limit = 100; + const all = await drizzle + .select() + .from(events) + .where( + and( + eq(events.runId, runId), + map(params?.sinceCursor, (c) => gt(events.eventId, c)) + ) + ) + .orderBy(events.eventId) + .limit(limit + 1); + const page = all.slice(0, limit); + const maxSlot = params?.maxSlot ?? 0; + const resolveData = params?.resolveData ?? 'all'; + return { + events: page + .filter((v) => (slotFromId(v.eventId) ?? 0) > maxSlot) + .map((v) => { + v.eventData ||= v.eventDataJson; + return stripEventDataRefs(EventSchema.parse(compact(v)), resolveData); + }), + cursor: page.at(-1)?.eventId ?? null, + hasMore: all.length > limit, + }; + } + + /** + * The 409 a caller gets when the slot it named turns out to belong to someone + * else, carrying the events it is missing so it can merge, replay and + * re-propose at a free position. + */ + async function slotConflict( + runId: string, + eventId: string, + params: CreateEventParams | undefined + ): Promise { + return new SlotConflictError( + `Slot ${slotFromId(eventId)} of run "${runId}" is already taken`, + { eventId, ...(await eventsAfterClaim(runId, params)) } + ); + } + return { async create(runId, data, params): Promise { let eventId: string | undefined; @@ -490,6 +562,20 @@ export function createEventsStorage(drizzle: Drizzle): Storage['events'] { // specVersion is always sent by the runtime, but we provide a fallback for safety const effectiveSpecVersion = data.specVersion ?? SPEC_VERSION_CURRENT; + // Whether this run numbers its events by slot. Decided from what was + // persisted, never from this request or this build, so a run stays in the + // mode it was created in for life — a run whose log holds ULID ids must + // never be handed a slot id, and vice versa. `run_created` is the one + // event that decides the mode instead of reading it; the resilient-start + // path below decides it too, on the request that creates the run. + let slotMode: boolean | undefined = + data.eventType === 'run_created' + ? usesSlotIdentity(effectiveSpecVersion) + : undefined; + // The run's highest event id, when it was read before the write. Seeds the + // allocator's first attempt so a probe is never made twice. + let seedHighestEventId: string | undefined; + // Track entity created/updated for EventResult let run: WorkflowRun | undefined; let step: Step | undefined; @@ -585,7 +671,13 @@ export function createEventsStorage(drizzle: Drizzle): Storage['events'] { .returning(); if (inserted) { - const runCreatedEventId = `wevt_${ulid()}`; + // We created the run, so this request also decided its mode. + slotMode = usesSlotIdentity(effectiveSpecVersion); + // A run's own `run_created` provably has nothing before it, so its + // slot needs no allocation. + const runCreatedEventId = slotMode + ? slotEventId(RUN_CREATED_SLOT) + : `wevt_${ulid()}`; await drizzle.insert(events).values({ runId: effectiveRunId, eventId: runCreatedEventId, @@ -631,7 +723,7 @@ export function createEventsStorage(drizzle: Drizzle): Storage['events'] { if (requiresNewerWorld(currentRun.specVersion)) { throw new RunNotSupportedError( currentRun.specVersion!, - SPEC_VERSION_CURRENT + SPEC_VERSION_MAX_SUPPORTED ); } @@ -651,6 +743,92 @@ export function createEventsStorage(drizzle: Drizzle): Storage['events'] { throw new WorkflowRunNotFoundError(effectiveRunId); } + // The run entity is the authority on the mode, and it can appear between + // the resilient-start insert above and this point: start() issues + // `run_created` and the queue send concurrently, so a delivery's + // `run_started` can arrive while the run is still being published. A stale + // "no" would number that one event with a ULID on an otherwise + // slot-numbered run, and the hole it leaves in the numbering costs the log + // its completeness proof for life. + if (currentRun && data.eventType !== 'run_created') { + slotMode = usesSlotIdentity(currentRun.specVersion); + } + if (slotMode === undefined) { + // step_completed and step_retrying skip the run read above. The log's + // own highest id reports the scheme, since a log holds ids of exactly + // one, and it is the probe the allocator needs anyway — so a + // slot-numbered run pays nothing extra for this query. + seedHighestEventId = await highestEventId(drizzle, effectiveRunId); + slotMode = isSlotId(seedHighestEventId ?? ''); + } + + // ============================================================ + // EVENT ID: the caller's slot claim, an allocated slot, or a ULID + // ============================================================ + // A slot-numbered run's ids name positions in its log, so an id is either + // claimed by a caller that holds the log (and is therefore asserting the + // log is complete up to that position) or allocated at write time for a + // caller that has no log — a step completion reporting in, a cancellation + // from an API call. + let claimedSlot: number | undefined; + if (params?.eventId !== undefined) { + if (!slotMode) { + throw new WorkflowWorldError( + `Event id "${params.eventId}" was supplied for run "${effectiveRunId}", whose events are not numbered by slot`, + { status: 400 } + ); + } + claimedSlot = slotFromId(params.eventId); + if (claimedSlot === undefined) { + throw new WorkflowWorldError( + `Event id "${params.eventId}" is not a slot id, and run "${effectiveRunId}" numbers its events by slot`, + { status: 400 } + ); + } + eventId = params.eventId; + if (await eventExists(drizzle, effectiveRunId, eventId)) { + // Reject a doomed claim before the materialization below creates the + // step, hook or wait this event will now never accompany. A caller + // that re-proposes at the next slot would otherwise collide with its + // own orphan and read that as "my write already landed". + throw await slotConflict(effectiveRunId, eventId, params); + } + } + + /** + * Runs one of the event writes below under this run's id discipline: in + * slot mode it places the event at the position the caller claimed or at + * the next free one, retrying a position lost to a concurrent writer; + * otherwise it mints a ULID. + */ + const publish = async ( + write: (ids: EventIds) => Promise + ): Promise => + slotMode + ? placeEvent(drizzle, effectiveRunId, { + ...(claimedSlot !== undefined ? { claimedSlot } : {}), + // A run's own `run_created` owns the first slot — provably, since + // nothing precedes it — so every other event allocates above it, + // even when that event is the first to arrive here. + minSlot: + data.eventType === 'run_created' + ? RUN_CREATED_SLOT + : RUN_CREATED_SLOT + 1, + seedHighestEventId, + onClaimTaken: () => + slotConflict(effectiveRunId, eventId as string, params), + write: (ids) => { + // Each attempt publishes under its own id, and the result and + // error messages below read it back from here. + eventId = ids.primary(); + return write(ids); + }, + }) + : write({ + primary: getEventId, + extra: async () => `wevt_${ulid()}`, + }); + // Lazy step start: a step_started carrying step-creation data // (stepName + input) may arrive with no prior step_created — it creates // the step on the fly (see the materialization block below). This @@ -676,17 +854,19 @@ export function createEventsStorage(drizzle: Drizzle): Storage['events'] { .limit(1); // Create the event (still record it) - const [value] = await drizzle - .insert(Schema.events) - .values({ - runId: effectiveRunId, - eventId: getEventId(), - correlationId: data.correlationId, - eventType: data.eventType, - eventData: 'eventData' in data ? data.eventData : undefined, - specVersion: effectiveSpecVersion, - }) - .returning({ createdAt: Schema.events.createdAt }); + const [value] = await publish((ids) => + drizzle + .insert(Schema.events) + .values({ + runId: effectiveRunId, + eventId: ids.primary(), + correlationId: data.correlationId, + eventType: data.eventType, + eventData: 'eventData' in data ? data.eventData : undefined, + specVersion: effectiveSpecVersion, + }) + .returning({ createdAt: Schema.events.createdAt }) + ); const result = { ...data, @@ -1192,41 +1372,52 @@ export function createEventsStorage(drizzle: Drizzle): Storage['events'] { // event INSERT behind that lock prevents a late step_started from being // ordered after a concurrent terminal event that already won the row. if (data.eventType === 'step_started') { - value = await drizzle.transaction(async (tx) => { - // Lazy step start: no prior step_created exists, but this - // step_started carries the step-creation data. The step INSERT is - // the ownership claim: only the caller that inserts the row gets to - // run the step body inline. - if (lazyStepStart && !validatedStep) { - const lazyData = data.eventData; - const [inserted] = await tx - .insert(Schema.steps) - .values({ - runId: effectiveRunId, - stepId: data.correlationId, - stepName: lazyData.stepName, - input: lazyData.input as SerializedContent, - status: 'pending', - attempt: 0, - specVersion: effectiveSpecVersion, - }) - .onConflictDoNothing() - .returning({ stepId: Schema.steps.stepId }); - - if (!inserted) { - throw new EntityConflictError( - `Step "${data.correlationId}" already created` - ); - } + // The whole transaction is the retry unit here: a step_started that + // loses its slot has to roll the step row and the synthetic + // step_created back with it, or the next attempt would trip its own + // orphaned step and read that as "a concurrent handler won the create". + value = await publish((ids) => + drizzle.transaction(async (tx) => { + // Lazy step start: no prior step_created exists, but this + // step_started carries the step-creation data. The step INSERT is + // the ownership claim: only the caller that inserts the row gets to + // run the step body inline. + if (lazyStepStart && !validatedStep) { + const lazyData = data.eventData; + const [inserted] = await tx + .insert(Schema.steps) + .values({ + runId: effectiveRunId, + stepId: data.correlationId, + stepName: lazyData.stepName, + input: lazyData.input as SerializedContent, + status: 'pending', + attempt: 0, + specVersion: effectiveSpecVersion, + }) + .onConflictDoNothing() + .returning({ stepId: Schema.steps.stepId }); + + if (!inserted) { + throw new EntityConflictError( + `Step "${data.correlationId}" already created` + ); + } - // Replay still needs to observe step_created before - // step_started. Because this synthetic event is in the same - // transaction as the lazy step row and step_started event, we - // cannot leave behind only one side of that materialization. - const stepCreatedEventId = `wevt_${ulid()}`; - await tx - .insert(events) - .values({ + // Replay still needs to observe a step_created at all: the + // client's step consumer sets hasCreatedEvent only on that event + // type. Which of the pair sorts first does not matter — the + // step_started consumer is a no-op — but leaving behind only one + // side of the materialization would, hence the shared + // transaction. + // + // It takes a position of its own, which on a claimed write is + // necessarily one this world allocated: a caller that defers a + // step_created cannot know it will be synthesized, so it never + // claims a slot for it. Losing that position rolls the transaction + // back for a retry, so the insert must not swallow the collision. + const stepCreatedEventId = await ids.extra(); + const insertStepCreated = tx.insert(events).values({ runId: effectiveRunId, eventId: stepCreatedEventId, correlationId: data.correlationId, @@ -1236,99 +1427,104 @@ export function createEventsStorage(drizzle: Drizzle): Storage['events'] { input: lazyData.input, }, specVersion: effectiveSpecVersion, - }) - .onConflictDoNothing(); - stepCreatedLazily = true; - } - - // Retried steps may be scheduled for later. Keep this check inside - // the transaction so the step_started write cannot slip past it. - if ( - validatedStep?.retryAfter && - validatedStep.retryAfter.getTime() > Date.now() - ) { - throw new TooEarlyError( - `Cannot start step "${data.correlationId}": retryAfter timestamp has not been reached yet`, - { - retryAfter: Math.ceil( - (validatedStep.retryAfter.getTime() - Date.now()) / 1000 - ), - } - ); - } + }); + await (slotMode + ? insertStepCreated + : insertStepCreated.onConflictDoNothing()); + stepCreatedLazily = true; + } - // The terminal-state guard is part of the UPDATE, not just the - // earlier validation read. That closes the race where another - // writer completes/fails the step between validation and start. - const [stepValue] = await tx - .update(Schema.steps) - .set({ - status: 'running', - attempt: sql`${Schema.steps.attempt} + 1`, - // Preserve the original first-start timestamp across retries or - // overlapping starts. - startedAt: sql`COALESCE(${Schema.steps.startedAt}, ${now.toISOString()})`, - retryAfter: null, - }) - .where( - and( - eq(Schema.steps.runId, effectiveRunId), - eq(Schema.steps.stepId, data.correlationId!), - notInArray(Schema.steps.status, terminalStepStatuses) - ) - ) - .returning(); + // Retried steps may be scheduled for later. Keep this check inside + // the transaction so the step_started write cannot slip past it. + if ( + validatedStep?.retryAfter && + validatedStep.retryAfter.getTime() > Date.now() + ) { + throw new TooEarlyError( + `Cannot start step "${data.correlationId}": retryAfter timestamp has not been reached yet`, + { + retryAfter: Math.ceil( + (validatedStep.retryAfter.getTime() - Date.now()) / 1000 + ), + } + ); + } - if (stepValue) { - step = deserializeStepError(compact(stepValue)); - } else { - const [existing] = await tx - .select({ status: Schema.steps.status }) - .from(Schema.steps) + // The terminal-state guard is part of the UPDATE, not just the + // earlier validation read. That closes the race where another + // writer completes/fails the step between validation and start. + const [stepValue] = await tx + .update(Schema.steps) + .set({ + status: 'running', + attempt: sql`${Schema.steps.attempt} + 1`, + // Preserve the original first-start timestamp across retries or + // overlapping starts. + startedAt: sql`COALESCE(${Schema.steps.startedAt}, ${now.toISOString()})`, + retryAfter: null, + }) .where( and( eq(Schema.steps.runId, effectiveRunId), - eq(Schema.steps.stepId, data.correlationId!) + eq(Schema.steps.stepId, data.correlationId!), + notInArray(Schema.steps.status, terminalStepStatuses) ) ) - .limit(1); - if (!existing) { - throw new WorkflowWorldError( - `Step "${data.correlationId}" not found` - ); + .returning(); + + if (stepValue) { + step = deserializeStepError(compact(stepValue)); + } else { + const [existing] = await tx + .select({ status: Schema.steps.status }) + .from(Schema.steps) + .where( + and( + eq(Schema.steps.runId, effectiveRunId), + eq(Schema.steps.stepId, data.correlationId!) + ) + ) + .limit(1); + if (!existing) { + throw new WorkflowWorldError( + `Step "${data.correlationId}" not found` + ); + } + if (isTerminalStepStatus(existing.status)) { + throw new EntityConflictError( + `Cannot modify step in terminal state "${existing.status}"` + ); + } } - if (isTerminalStepStatus(existing.status)) { + + // A ULID-numbered step_started takes its id only after the guarded + // step UPDATE has acquired and passed the row lock. Without a + // sequence, this is the local ordering guarantee we can provide: a + // writer blocked on the step row will not carry an older event id + // into a later insert. A slot id is exempt — it names a position in + // the log, not a time — and the caller may have claimed it already. + const stepStartedEventId = ids.primary(); + eventId = stepStartedEventId; + const [eventValue] = await tx + .insert(events) + .values({ + runId: effectiveRunId, + eventId: stepStartedEventId, + correlationId: data.correlationId, + eventType: data.eventType, + eventData: storedEventData, + specVersion: effectiveSpecVersion, + }) + .returning({ createdAt: events.createdAt }); + + if (!eventValue) { throw new EntityConflictError( - `Cannot modify step in terminal state "${existing.status}"` + `Event ${stepStartedEventId} could not be created` ); } - } - - // Allocate the step_started ULID only after the guarded step UPDATE - // has acquired and passed the row lock. Without a sequence, this is - // the local ordering guarantee we can provide: a writer blocked on - // the step row will not carry an older event id into a later insert. - const stepStartedEventId = `wevt_${ulid()}`; - eventId = stepStartedEventId; - const [eventValue] = await tx - .insert(events) - .values({ - runId: effectiveRunId, - eventId: stepStartedEventId, - correlationId: data.correlationId, - eventType: data.eventType, - eventData: storedEventData, - specVersion: effectiveSpecVersion, - }) - .returning({ createdAt: events.createdAt }); - - if (!eventValue) { - throw new EntityConflictError( - `Event ${stepStartedEventId} could not be created` - ); - } - return eventValue; - }); + return eventValue; + }) + ); } // Handle step_completed event: update step status @@ -1531,20 +1727,21 @@ export function createEventsStorage(drizzle: Drizzle): Storage['events'] { token: eventData.token, conflictingRunId: existingHook.runId, }; + const [conflictValue] = await publish((ids) => + drizzle + .insert(events) + .values({ + runId: effectiveRunId, + eventId: ids.primary(), + correlationId: data.correlationId, + eventType: 'hook_conflict', + eventData: conflictEventData, + specVersion: effectiveSpecVersion, + }) + .returning({ createdAt: events.createdAt }) + ); const conflictEventId = getEventId(); - const [conflictValue] = await drizzle - .insert(events) - .values({ - runId: effectiveRunId, - eventId: conflictEventId, - correlationId: data.correlationId, - eventType: 'hook_conflict', - eventData: conflictEventData, - specVersion: effectiveSpecVersion, - }) - .returning({ createdAt: events.createdAt }); - if (!conflictValue) { throw new EntityConflictError( `Event ${conflictEventId} could not be created` @@ -1622,47 +1819,49 @@ export function createEventsStorage(drizzle: Drizzle): Storage['events'] { // guarded UPDATE linearizes against a concurrent terminal step // event. if (data.eventType === 'hook_received') { - value = await drizzle.transaction(async (tx) => { - const [runRow] = await tx - .select({ status: Schema.runs.status }) - .from(Schema.runs) - .where(eq(Schema.runs.runId, effectiveRunId)) - .for('update') - .limit(1); - if (!runRow) { - throw new WorkflowRunNotFoundError(effectiveRunId); - } - if (isTerminalWorkflowRunStatus(runRow.status)) { - throw new RunExpiredError( - `Workflow run "${effectiveRunId}" is already in terminal state "${runRow.status}"` - ); - } + value = await publish((ids) => + drizzle.transaction(async (tx) => { + const [runRow] = await tx + .select({ status: Schema.runs.status }) + .from(Schema.runs) + .where(eq(Schema.runs.runId, effectiveRunId)) + .for('update') + .limit(1); + if (!runRow) { + throw new WorkflowRunNotFoundError(effectiveRunId); + } + if (isTerminalWorkflowRunStatus(runRow.status)) { + throw new RunExpiredError( + `Workflow run "${effectiveRunId}" is already in terminal state "${runRow.status}"` + ); + } - // Allocate the ULID only after the row lock is acquired, - // matching step_started's ordering guarantee: a writer blocked - // on the run row must not carry an older event id into a later - // insert. - const hookReceivedEventId = `wevt_${ulid()}`; - eventId = hookReceivedEventId; - const [eventValue] = await tx - .insert(events) - .values({ - runId: effectiveRunId, - eventId: hookReceivedEventId, - correlationId: data.correlationId, - eventType: data.eventType, - eventData: storedEventData, - specVersion: effectiveSpecVersion, - }) - .returning({ createdAt: events.createdAt }); + // Take the ULID only after the row lock is acquired, matching + // step_started's ordering guarantee: a writer blocked on the run + // row must not carry an older event id into a later insert. A slot + // id names a position rather than a time, so it is exempt. + const hookReceivedEventId = ids.primary(); + eventId = hookReceivedEventId; + const [eventValue] = await tx + .insert(events) + .values({ + runId: effectiveRunId, + eventId: hookReceivedEventId, + correlationId: data.correlationId, + eventType: data.eventType, + eventData: storedEventData, + specVersion: effectiveSpecVersion, + }) + .returning({ createdAt: events.createdAt }); - if (!eventValue) { - throw new EntityConflictError( - `Event ${hookReceivedEventId} could not be created` - ); - } - return eventValue; - }); + if (!eventValue) { + throw new EntityConflictError( + `Event ${hookReceivedEventId} could not be created` + ); + } + return eventValue; + }) + ); } // Handle wait_created event: create wait entity @@ -1748,17 +1947,23 @@ export function createEventsStorage(drizzle: Drizzle): Storage['events'] { try { if (!value) { - [value] = await drizzle - .insert(events) - .values({ - runId: effectiveRunId, - eventId: getEventId(), - correlationId: data.correlationId, - eventType: data.eventType, - eventData: storedEventData, - specVersion: effectiveSpecVersion, - }) - .returning({ createdAt: events.createdAt }); + // Only the event row is retried here: the entity this event describes + // was materialized above, outside any transaction, and re-inserting + // the event at a higher position leaves the log dense and still + // consistent with that entity. + [value] = await publish((ids) => + drizzle + .insert(events) + .values({ + runId: effectiveRunId, + eventId: ids.primary(), + correlationId: data.correlationId, + eventType: data.eventType, + eventData: storedEventData, + specVersion: effectiveSpecVersion, + }) + .returning({ createdAt: events.createdAt }) + ); } } catch (err) { // Translate unique-violation on the correlated-event partial index diff --git a/packages/world-postgres/test/slot-identity.test.ts b/packages/world-postgres/test/slot-identity.test.ts new file mode 100644 index 0000000000..d8b0578a7d --- /dev/null +++ b/packages/world-postgres/test/slot-identity.test.ts @@ -0,0 +1,375 @@ +import { execSync } from 'node:child_process'; +import { PostgreSqlContainer } from '@testcontainers/postgresql'; +import { SlotConflictError } from '@workflow/errors'; +import { + FIRST_SLOT, + maxSlotOf, + SPEC_VERSION_CURRENT, + SPEC_VERSION_SLOT_IDENTITY, + slotEventId, + slotFromId, +} from '@workflow/world'; +import { Pool } from 'pg'; +import { + afterAll, + beforeAll, + beforeEach, + describe, + expect, + test, +} from 'vitest'; +import { createClient } from '../src/drizzle/index.js'; +import { createEventsStorage } from '../src/storage.js'; + +describe('Slot identity (Postgres integration)', () => { + if (process.platform === 'win32') { + test.skip('skipped on Windows since it relies on a docker container', () => {}); + return; + } + + let container: Awaited>; + let pool: Pool; + let events: ReturnType; + + beforeAll(async () => { + container = await new PostgreSqlContainer('postgres:15-alpine').start(); + const dbUrl = container.getConnectionUri(); + process.env.DATABASE_URL = dbUrl; + process.env.WORKFLOW_POSTGRES_URL = dbUrl; + execSync('pnpm db:push', { + stdio: 'inherit', + cwd: process.cwd(), + env: process.env, + }); + // Contention is the point of these tests, so the pool has to be able to + // hold every writer of a burst at once. + pool = new Pool({ connectionString: dbUrl, max: 20 }); + events = createEventsStorage(createClient(pool)); + }, 120_000); + + beforeEach(async () => { + await pool.query( + 'TRUNCATE TABLE workflow.workflow_events, workflow.workflow_steps, workflow.workflow_hooks, workflow.workflow_runs RESTART IDENTITY CASCADE' + ); + }); + + afterAll(async () => { + await pool.end(); + await container.stop(); + }); + + /** Start a run whose events are numbered by slot, and return its id. */ + async function newSlotRun(): Promise { + const result = await events.create(null, { + eventType: 'run_created', + specVersion: SPEC_VERSION_SLOT_IDENTITY, + eventData: { + deploymentId: 'dpl_test', + workflowName: 'test-workflow', + input: new Uint8Array(), + }, + }); + if (!result.run) { + throw new Error('Expected run to be created'); + } + return result.run.runId; + } + + function eventsOf(runId: string) { + // The page size is explicit: the default would silently truncate a fan-out + // and make a dense log look sparse. + return events.list({ runId, pagination: { limit: 500 } }); + } + + /** The slots of the run's log, in list order. */ + async function slotsOf(runId: string): Promise { + const { data } = await eventsOf(runId); + return data.map((event) => slotFromId(event.eventId) ?? -1); + } + + function ascending(slots: number[]): number[] { + return [...slots].sort((a, b) => a - b); + } + + function denseFrom(count: number): number[] { + return Array.from({ length: count }, (_, index) => FIRST_SLOT + index); + } + + async function createStep( + runId: string, + stepId: string, + eventId?: string + ): Promise { + const result = await events.create( + runId, + { + eventType: 'step_created', + specVersion: SPEC_VERSION_SLOT_IDENTITY, + correlationId: stepId, + eventData: { stepName: 'a-step', input: new Uint8Array() }, + }, + eventId === undefined ? undefined : { eventId } + ); + if (!result.event) { + throw new Error('Expected an event'); + } + return result.event.eventId; + } + + describe('numbering', () => { + test('puts run_created in the first slot', async () => { + const runId = await newSlotRun(); + await expect(slotsOf(runId)).resolves.toEqual([FIRST_SLOT]); + }); + + test('allocates dense slots for writers that hold no log', async () => { + // A step completion reporting in, a cancellation from an API call: the + // caller has no event log, so the world numbers the event for it. + const runId = await newSlotRun(); + await createStep(runId, 'step_a'); + await createStep(runId, 'step_b'); + await expect(slotsOf(runId)).resolves.toEqual(denseFrom(3)); + }); + + test('honours a slot the caller claims', async () => { + const runId = await newSlotRun(); + const eventId = await createStep(runId, 'step_a', slotEventId(2)); + expect(eventId).toBe(slotEventId(2)); + await expect(slotsOf(runId)).resolves.toEqual(denseFrom(2)); + }); + + test('numbers a ULID-mode run the way it always did', async () => { + const created = await events.create(null, { + eventType: 'run_created', + specVersion: SPEC_VERSION_CURRENT, + eventData: { + deploymentId: 'dpl_test', + workflowName: 'test-workflow', + input: new Uint8Array(), + }, + }); + const runId = created.run?.runId as string; + const eventId = await createStep(runId, 'step_a'); + expect(eventId).toMatch(/^wevt_/); + expect(slotFromId(eventId)).toBeUndefined(); + }); + + test('gives a lazy step start two consecutive slots', async () => { + // One request, two events: the step_started the caller sent and the + // step_created it deferred. Which sorts first does not matter — only + // step_created flips the client's hasCreatedEvent — but both have to + // land, and neither may leave a hole. + const runId = await newSlotRun(); + const started = await events.create(runId, { + eventType: 'step_started', + specVersion: SPEC_VERSION_SLOT_IDENTITY, + correlationId: 'step_a', + eventData: { stepName: 'a-step', input: new Uint8Array() }, + }); + expect(slotFromId(started.event?.eventId ?? '')).toBe(2); + const { data } = await eventsOf(runId); + expect( + data.map((event) => `${slotFromId(event.eventId)} ${event.eventType}`) + ).toEqual(['1 run_created', '2 step_started', '3 step_created']); + }); + + test('allocates the deferred step_created above a claimed slot', async () => { + // A caller that defers a step_created cannot know it will be synthesized, + // so it never claims a slot for it — and the slot it did claim is its own. + const runId = await newSlotRun(); + const started = await events.create( + runId, + { + eventType: 'step_started', + specVersion: SPEC_VERSION_SLOT_IDENTITY, + correlationId: 'step_a', + eventData: { stepName: 'a-step', input: new Uint8Array() }, + }, + { eventId: slotEventId(2) } + ); + expect(started.event?.eventId).toBe(slotEventId(2)); + expect(ascending(await slotsOf(runId))).toEqual(denseFrom(3)); + }); + + test('numbers events of runs it never created', async () => { + // `step_completed` and `step_retrying` deliberately skip the run read, so + // the mode comes from the log rather than from a run row in hand. + const runId = await newSlotRun(); + await createStep(runId, 'step_a'); + const completed = await events.create(runId, { + eventType: 'step_completed', + specVersion: SPEC_VERSION_CURRENT, + correlationId: 'step_a', + eventData: { output: new Uint8Array() }, + }); + expect(slotFromId(completed.event?.eventId ?? '')).toBe(3); + }); + }); + + describe('contention', () => { + // The primary key is the authority, and a writer that loses a position is + // retried at one that is still free. Nothing here may leave a hole: density + // is what lets a reader prove its copy of a log is complete. + for (const writers of [2, 8, 50]) { + test(`keeps ${writers} concurrent writers dense`, async () => { + const runId = await newSlotRun(); + const ids = await Promise.all( + Array.from({ length: writers }, (_, index) => + createStep(runId, `step_${index}`) + ) + ); + expect(new Set(ids).size).toBe(writers); + expect(ascending(await slotsOf(runId))).toEqual(denseFrom(writers + 1)); + }, 60_000); + } + + test('proves completeness: the highest slot is the event count', async () => { + const runId = await newSlotRun(); + await Promise.all( + Array.from({ length: 5 }, (_, index) => + createStep(runId, `step_${index}`) + ) + ); + const { data } = await eventsOf(runId); + expect(maxSlotOf(data)).toBe(data.length); + }); + + test('leaves no hole behind a rejected write', async () => { + // The rejected op's slot sits below its concurrent sibling's, and a hole + // below a published event can never be filled. + const runId = await newSlotRun(); + const [rejected, accepted] = await Promise.allSettled([ + events.create(runId, { + eventType: 'step_completed', + specVersion: SPEC_VERSION_SLOT_IDENTITY, + correlationId: 'step_never_created', + eventData: { output: new Uint8Array() }, + }), + createStep(runId, 'step_a'), + ]); + expect(rejected.status).toBe('rejected'); + expect(accepted.status).toBe('fulfilled'); + await createStep(runId, 'step_b'); + expect(ascending(await slotsOf(runId))).toEqual(denseFrom(3)); + }); + }); + + describe('mode is pinned to the run', () => { + test('rejects a slot id claimed on a ULID-numbered run', async () => { + const created = await events.create(null, { + eventType: 'run_created', + specVersion: SPEC_VERSION_CURRENT, + eventData: { + deploymentId: 'dpl_test', + workflowName: 'test-workflow', + input: new Uint8Array(), + }, + }); + const runId = created.run?.runId as string; + await expect(createStep(runId, 'step_a', slotEventId(2))).rejects.toThrow( + /not numbered by slot/ + ); + }); + + test('rejects a ULID id claimed on a slot-numbered run', async () => { + const runId = await newSlotRun(); + await expect( + createStep(runId, 'step_a', 'evnt_01K5Z0000000000000000000AA') + ).rejects.toThrow(/not a slot id/); + }); + + test('ignores the spec version of later requests', async () => { + // A run is in exactly one mode for life; only what was persisted decides. + const runId = await newSlotRun(); + const result = await events.create(runId, { + eventType: 'step_created', + specVersion: SPEC_VERSION_CURRENT, + correlationId: 'step_a', + eventData: { stepName: 'a-step', input: new Uint8Array() }, + }); + expect(slotFromId(result.event?.eventId ?? '')).toBe(2); + }); + }); + + describe('conflict', () => { + test('reports the events the loser is missing', async () => { + const runId = await newSlotRun(); + // Out of band: something else takes the slot this caller was about to + // claim, so the caller's log is provably missing an event. + await createStep(runId, 'step_out_of_band'); + + const conflict = await events + .create( + runId, + { + eventType: 'step_created', + specVersion: SPEC_VERSION_SLOT_IDENTITY, + correlationId: 'step_a', + eventData: { stepName: 'a-step', input: new Uint8Array() }, + }, + { eventId: slotEventId(2), maxSlot: 1 } + ) + .catch((error: unknown) => error); + + expect(SlotConflictError.is(conflict)).toBe(true); + const slotConflict = conflict as SlotConflictError; + expect(slotConflict.status).toBe(409); + expect(slotConflict.eventId).toBe(slotEventId(2)); + expect(slotConflict.events?.map((event) => event.eventId)).toEqual([ + slotEventId(2), + ]); + }); + + test('excludes events the loser already holds from the delta', async () => { + const runId = await newSlotRun(); + await createStep(runId, 'step_one'); + await createStep(runId, 'step_two'); + + const conflict = await events + .create( + runId, + { + eventType: 'step_created', + specVersion: SPEC_VERSION_SLOT_IDENTITY, + correlationId: 'step_a', + eventData: { stepName: 'a-step', input: new Uint8Array() }, + }, + { eventId: slotEventId(2), maxSlot: 2 } + ) + .catch((error: unknown) => error); + + // Slots 1 and 2 are at or below what the caller had; only 3 is news. + expect( + (conflict as SlotConflictError).events?.map((event) => event.eventId) + ).toEqual([slotEventId(3)]); + }); + + test('lets the loser re-propose at the next free slot', async () => { + const runId = await newSlotRun(); + await createStep(runId, 'step_out_of_band'); + await expect(createStep(runId, 'step_a', slotEventId(2))).rejects.toThrow( + SlotConflictError + ); + // Merging the delta moves the caller's own numbering forward by one. + const eventId = await createStep(runId, 'step_a', slotEventId(3)); + expect(eventId).toBe(slotEventId(3)); + await expect(slotsOf(runId)).resolves.toEqual(denseFrom(3)); + }); + + test('materializes nothing for a claim that is already taken', async () => { + // The re-post is what the guard protects: a step row left behind by the + // losing attempt would make the retry trip its own orphan and read that + // as "a concurrent handler won the create". + const runId = await newSlotRun(); + await createStep(runId, 'step_out_of_band'); + await expect(createStep(runId, 'step_a', slotEventId(2))).rejects.toThrow( + SlotConflictError + ); + const { rows } = await pool.query( + 'SELECT step_id FROM workflow.workflow_steps WHERE run_id = $1', + [runId] + ); + expect(rows.map((row) => row.step_id)).toEqual(['step_out_of_band']); + }); + }); +}); diff --git a/packages/world/src/index.ts b/packages/world/src/index.ts index 7317b340ba..5a8c185f2a 100644 --- a/packages/world/src/index.ts +++ b/packages/world/src/index.ts @@ -113,9 +113,12 @@ export { export type { SpecVersion } from './spec-version.js'; export { isLegacySpecVersion, + mintedSpecVersion, requiresNewerWorld, + SLOT_IDENTITY_ENV_VAR, SPEC_VERSION_CURRENT, SPEC_VERSION_LEGACY, + SPEC_VERSION_MAX_SUPPORTED, SPEC_VERSION_SLOT_IDENTITY, SPEC_VERSION_SUPPORTS_ATTRIBUTES, SPEC_VERSION_SUPPORTS_CBOR_QUEUE_TRANSPORT, diff --git a/packages/world/src/slot-identity.test.ts b/packages/world/src/slot-identity.test.ts index c99d2aa930..acdfada627 100644 --- a/packages/world/src/slot-identity.test.ts +++ b/packages/world/src/slot-identity.test.ts @@ -1,5 +1,6 @@ import { ulid } from 'ulid'; import { describe, expect, it } from 'vitest'; +import { z } from 'zod'; import { FIRST_SLOT, isSlotId, @@ -17,9 +18,9 @@ describe('slotIdBody', () => { expect(body).toHaveLength(SLOT_ID_WIDTH); expect(`evnt_${body}`).toHaveLength(`evnt_${ulid()}`.length); // Crockford base32 starts with the decimal digits, so the padded body - // parses as a ULID — this is what keeps every existing schema, sort key and - // range fence working unchanged. - expect(ulidToDate(body)).not.toBeNull(); + // satisfies the ULID syntax — this is what keeps every existing schema, + // sort key and range fence working unchanged. + expect(z.string().ulid().safeParse(body).success).toBe(true); }); it('orders lexicographically by slot at a fixed width', () => { @@ -60,6 +61,24 @@ describe('slotFromId', () => { }); }); +describe('a slot carries no timestamp', () => { + it('reports no time rather than epoch 0', () => { + // Passing the ULID syntax check is what makes a slot portable; decoding a + // *time* out of one is always a bug. Two that this guards: the sandbox + // clock is set from the events it consumes, so epoch 0 would rewind a + // replaying workflow's `Date.now()` to 1970; and world-local prefilters + // cursor pagination on the time in the filename, so epoch 0 would hide + // every slot-numbered event from an ascending page. + expect(ulidToDate(slotIdBody(FIRST_SLOT))).toBeNull(); + expect(ulidToDate(slotEventId(FIRST_SLOT))).toBeNull(); + expect(ulidToDate(slotIdBody(123_456))).toBeNull(); + }); + + it('still reads the time out of a ULID', () => { + expect(ulidToDate(ulid())?.getTime()).toBeGreaterThan(0); + }); +}); + describe('maxSlotOf', () => { it('finds the highest slot regardless of position', () => { // A log is merged from several loads and is not sorted, so the last element diff --git a/packages/world/src/spec-version.test.ts b/packages/world/src/spec-version.test.ts index 880c43180d..f21596f951 100644 --- a/packages/world/src/spec-version.test.ts +++ b/packages/world/src/spec-version.test.ts @@ -1,9 +1,13 @@ import { describe, expect, it } from 'vitest'; import { isLegacySpecVersion, + mintedSpecVersion, requiresNewerWorld, + SLOT_IDENTITY_ENV_VAR, SPEC_VERSION_CURRENT, SPEC_VERSION_LEGACY, + SPEC_VERSION_MAX_SUPPORTED, + SPEC_VERSION_SLOT_IDENTITY, SPEC_VERSION_SUPPORTS_ATTRIBUTES, SPEC_VERSION_SUPPORTS_COMPRESSION, } from './spec-version.js'; @@ -13,10 +17,21 @@ describe('spec version constants', () => { expect(SPEC_VERSION_CURRENT).toBe(SPEC_VERSION_SUPPORTS_COMPRESSION); expect(SPEC_VERSION_SUPPORTS_COMPRESSION).toBe(5); }); + + it('can read a newer spec version than it mints', () => { + // Slot identity is readable by every world before any world mints it, so + // that turning it on for new runs cannot make those same worlds reject + // them. Once slots are the default the two constants coincide again. + expect(SPEC_VERSION_MAX_SUPPORTED).toBe(SPEC_VERSION_SLOT_IDENTITY); + expect(SPEC_VERSION_MAX_SUPPORTED).toBeGreaterThanOrEqual( + SPEC_VERSION_CURRENT + ); + }); }); describe('requiresNewerWorld', () => { - it('accepts runs at or below the current spec version', () => { + it('accepts runs at or below the newest readable spec version', () => { + expect(requiresNewerWorld(SPEC_VERSION_MAX_SUPPORTED)).toBe(false); expect(requiresNewerWorld(SPEC_VERSION_CURRENT)).toBe(false); expect(requiresNewerWorld(SPEC_VERSION_SUPPORTS_ATTRIBUTES)).toBe(false); expect(requiresNewerWorld(SPEC_VERSION_LEGACY)).toBe(false); @@ -24,13 +39,20 @@ describe('requiresNewerWorld', () => { expect(requiresNewerWorld(null)).toBe(false); }); - it('rejects runs newer than the current spec version', () => { + it('accepts a slot-identity run', () => { + // Gates the flag rollout: a world that rejected spec-6 would reject the + // runs it had just stamped spec-6 itself, at their first event after + // run_created. + expect(requiresNewerWorld(SPEC_VERSION_SLOT_IDENTITY)).toBe(false); + }); + + it('rejects runs newer than the newest readable spec version', () => { // This is the contract that protects older SDKs from compressed // payloads they cannot decode: a spec-5 run read by an SDK whose - // SPEC_VERSION_CURRENT is 4 fails this check up front (with - // RunNotSupportedError at the storage layer) instead of failing on - // individual compressed payloads. - expect(requiresNewerWorld(SPEC_VERSION_CURRENT + 1)).toBe(true); + // ceiling is 4 fails this check up front (with RunNotSupportedError at + // the storage layer) instead of failing on individual compressed + // payloads. + expect(requiresNewerWorld(SPEC_VERSION_MAX_SUPPORTED + 1)).toBe(true); }); it('simulates a v4 reader rejecting a compression-era run', () => { @@ -51,3 +73,33 @@ describe('isLegacySpecVersion', () => { expect(isLegacySpecVersion(5)).toBe(false); }); }); + +describe('mintedSpecVersion', () => { + it('mints the current version by default', () => { + expect(mintedSpecVersion({})).toBe(SPEC_VERSION_CURRENT); + }); + + it('mints slot identity when the flag is set', () => { + for (const value of ['1', 'true']) { + expect(mintedSpecVersion({ [SLOT_IDENTITY_ENV_VAR]: value })).toBe( + SPEC_VERSION_SLOT_IDENTITY + ); + } + }); + + it('treats any other value as off', () => { + // An unset-but-present variable is the shape a shell leaves behind, and it + // must not silently switch a deployment's event identity scheme. + for (const value of ['', '0', 'false', 'yes']) { + expect(mintedSpecVersion({ [SLOT_IDENTITY_ENV_VAR]: value })).toBe( + SPEC_VERSION_CURRENT + ); + } + }); + + it('mints nothing a world cannot read', () => { + expect( + requiresNewerWorld(mintedSpecVersion({ [SLOT_IDENTITY_ENV_VAR]: '1' })) + ).toBe(false); + }); +}); diff --git a/packages/world/src/spec-version.ts b/packages/world/src/spec-version.ts index d9be743be6..a86f4f3e90 100644 --- a/packages/world/src/spec-version.ts +++ b/packages/world/src/spec-version.ts @@ -42,20 +42,60 @@ export const SPEC_VERSION_SUPPORTS_COMPRESSION = 5 as SpecVersion; * persisted `specVersion`, never from the build. A run started under ULID * correlation ids and replayed by a slot-capable build would otherwise propose * `step_…001` where its log holds `step_01K…`, matching no existing entity. - * - * Deliberately not `SPEC_VERSION_CURRENT` yet: `requiresNewerWorld()` is what - * makes a world reject runs it cannot read, so bumping current before the - * worlds can allocate slots would have them reject their own new runs. */ export const SPEC_VERSION_SLOT_IDENTITY = 6 as SpecVersion; /** * Current spec version (event-sourced architecture with native attributes * and compressed payloads). + * + * This is the version new runs are stamped with, which is a *lower* bar than + * the newest version this build can read — see + * {@link SPEC_VERSION_MAX_SUPPORTED}. Slot identity ships behind a flag, so it + * is readable everywhere before it is minted anywhere. */ export const SPEC_VERSION_CURRENT = SPEC_VERSION_SUPPORTS_COMPRESSION as SpecVersion; +/** + * Newest spec version this build can read. Runs above it are rejected outright + * by {@link requiresNewerWorld} rather than misread. + * + * Distinct from {@link SPEC_VERSION_CURRENT} because a world has to be able to + * read a version before anything may mint it: the flag that turns slot identity + * on for new runs would otherwise make every world reject the runs it had just + * created. Worlds opt into minting individually, via the `specVersion` they + * declare. + */ +export const SPEC_VERSION_MAX_SUPPORTED = + SPEC_VERSION_SLOT_IDENTITY as SpecVersion; + +/** + * Environment variable that opts new runs into slot identity. + * + * Read per `createWorld()` call rather than at module load, so a test or a + * single process can create worlds in both modes. + */ +export const SLOT_IDENTITY_ENV_VAR = 'WORKFLOW_SLOT_IDENTITY'; + +/** + * The spec version a world should stamp on the runs it creates: slot identity + * when {@link SLOT_IDENTITY_ENV_VAR} is set, otherwise + * {@link SPEC_VERSION_CURRENT}. + * + * Every world reads runs up to {@link SPEC_VERSION_MAX_SUPPORTED} whatever this + * returns, so turning the flag on in one place does not make the runs it creates + * unreadable elsewhere. + */ +export function mintedSpecVersion( + env: Record = process.env +): SpecVersion { + const value = env[SLOT_IDENTITY_ENV_VAR]; + return value === '1' || value === 'true' + ? SPEC_VERSION_SLOT_IDENTITY + : SPEC_VERSION_CURRENT; +} + /** * Check if a spec version is legacy (<= SPEC_VERSION_LEGACY or undefined). * Legacy runs require different handling - they use direct entity mutation @@ -73,7 +113,7 @@ export function isLegacySpecVersion(v: number | undefined | null): boolean { } /** - * Check if a spec version requires a newer world (> SPEC_VERSION_CURRENT). + * Check if a spec version requires a newer world (> SPEC_VERSION_MAX_SUPPORTED). * This happens when a run was created by a newer SDK version. * * @param v - The spec version number, or undefined/null for legacy runs @@ -81,7 +121,7 @@ export function isLegacySpecVersion(v: number | undefined | null): boolean { */ export function requiresNewerWorld(v: number | undefined | null): boolean { if (v === undefined || v === null) return false; - return v > SPEC_VERSION_CURRENT; + return v > SPEC_VERSION_MAX_SUPPORTED; } /** diff --git a/packages/world/src/ulid.ts b/packages/world/src/ulid.ts index 1ee1b7b47c..cc893834c3 100644 --- a/packages/world/src/ulid.ts +++ b/packages/world/src/ulid.ts @@ -1,5 +1,6 @@ import { decodeTime } from 'ulid'; import { z } from 'zod'; +import { isSlotId } from './slot-identity.js'; const UlidSchema = z.string().ulid(); @@ -36,8 +37,19 @@ export const DEFAULT_TIMESTAMP_THRESHOLD_MS = /** * Extracts a Date from a ULID string, or null if the string is not a valid ULID. + * + * Slot ids are not ULIDs even though they pass the ULID *syntax* check: their + * body is all decimal digits, which Crockford base32 accepts, and it would + * decode to a timestamp of epoch 0 instead of failing. A slot encodes a + * position, not a time, so it is reported here as having no time at all — + * callers must read the object's own `createdAt`. Silently returning 1970 + * instead would, among other things, rewind a replaying workflow's clock and + * make cursor pagination skip every slot-numbered event. */ export function ulidToDate(maybeUlid: string): Date | null { + if (isSlotId(maybeUlid)) { + return null; + } const ulid = UlidSchema.safeParse(maybeUlid); if (!ulid.success) { return null; From 4cbd4f94f0f9027b9c344039bcddb445a2ad48c5 Mon Sep 17 00:00:00 2001 From: Peter Wielander Date: Thu, 30 Jul 2026 21:18:31 -0700 Subject: [PATCH 06/35] chore: sort imports in events-v4.test.ts for Biome organizeImports --- packages/world-vercel/src/events-v4.test.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/world-vercel/src/events-v4.test.ts b/packages/world-vercel/src/events-v4.test.ts index 2139d8c3eb..5b541e8258 100644 --- a/packages/world-vercel/src/events-v4.test.ts +++ b/packages/world-vercel/src/events-v4.test.ts @@ -7,9 +7,9 @@ import { WorkflowWorldError, } from '@workflow/errors'; import { + SPEC_VERSION_SLOT_IDENTITY, slotEventId, slotIdBody, - SPEC_VERSION_SLOT_IDENTITY, } from '@workflow/world'; import { decode, encode } from 'cbor-x'; import { MockAgent } from 'undici'; From 2ce3cb12ac1703a197a32bc7fd9b220b56e3caf6 Mon Sep 17 00:00:00 2001 From: Peter Wielander Date: Thu, 30 Jul 2026 21:48:59 -0700 Subject: [PATCH 07/35] feat(worlds): mint slot identity by default, including from world-vercel MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit `mintedSpecVersion()` now returns spec 6 unless `WORKFLOW_SLOT_IDENTITY` is explicitly `0`/`false`, and world-vercel reads it instead of hardcoding spec 5. world-vercel was the one World that never called it, so a deployment could not mint a slot-numbered run however the flag was set — the flag only reached the Local and Postgres Worlds. Every World still reads both schemes, and mode stays pinned to a run's persisted specVersion, so flipping the default only changes how runs created from here on are numbered. --- .changeset/slot-identity-default-on.md | 8 +++++ .../docs/v5/configuration/runtime-tuning.mdx | 6 ++-- packages/world-local/src/index.ts | 6 ++-- packages/world-postgres/src/index.ts | 6 ++-- packages/world-vercel/src/index.ts | 11 ++++--- packages/world/src/spec-version.test.ts | 23 +++++++-------- packages/world/src/spec-version.ts | 29 +++++++++---------- 7 files changed, 49 insertions(+), 40 deletions(-) create mode 100644 .changeset/slot-identity-default-on.md diff --git a/.changeset/slot-identity-default-on.md b/.changeset/slot-identity-default-on.md new file mode 100644 index 0000000000..55e3973586 --- /dev/null +++ b/.changeset/slot-identity-default-on.md @@ -0,0 +1,8 @@ +--- +'@workflow/world': minor +'@workflow/world-vercel': minor +'@workflow/world-local': minor +'@workflow/world-postgres': minor +--- + +Number new runs' events by position by default, in every World including the Vercel one. Set `WORKFLOW_SLOT_IDENTITY=0` to keep minting ULID event ids. diff --git a/docs/content/docs/v5/configuration/runtime-tuning.mdx b/docs/content/docs/v5/configuration/runtime-tuning.mdx index 110e569d5f..ba4a05b397 100644 --- a/docs/content/docs/v5/configuration/runtime-tuning.mdx +++ b/docs/content/docs/v5/configuration/runtime-tuning.mdx @@ -50,12 +50,12 @@ For example, a workflow can run a 10-minute inline step even with `WORKFLOW_REPL ### `WORKFLOW_SLOT_IDENTITY` -- Default: disabled +- Default: enabled - Numbers a new run's events by position instead of by ULID: `evnt_…001` is the run's first event, `evnt_…002` its second, with no gaps. A reader can then prove its copy of the log is complete, because the highest number is the event count. - Contention becomes explicit rather than silent. Two writers proposing the same position cannot both win: the loser gets a 409 ([`SlotConflictError`](/docs/api-reference/workflow-errors/slot-conflict-error)) carrying the events it was missing, and the runtime merges them, replays, and re-proposes at the next free position. -- Applies only to runs created while it is set. A run keeps the identity scheme it was created with for life, so turning the flag on or off never affects runs already in flight. +- Applies only to runs created while it is enabled. A run keeps the identity scheme it was created with for life, so turning the flag on or off never affects runs already in flight. - Requires a World that supports it. A World that does not rejects the run outright rather than mis-numbering its events. -- Set `1` or `true` to enable. +- Set `0` or `false` to disable, which numbers new runs by ULID as before. ## Inline execution diff --git a/packages/world-local/src/index.ts b/packages/world-local/src/index.ts index 2014a75652..aa77546732 100644 --- a/packages/world-local/src/index.ts +++ b/packages/world-local/src/index.ts @@ -72,9 +72,9 @@ export function createWorld(args?: Partial): LocalWorld { ); const recoverActiveRuns = resolveRecoverActiveRuns(mergedConfig); return { - // What this world stamps on new runs, which is not the newest version it - // can read: slot identity is readable everywhere and minted only where - // WORKFLOW_SLOT_IDENTITY is set. + // What this world stamps on new runs: slot identity, unless + // WORKFLOW_SLOT_IDENTITY switches it off. Every world reads both schemes + // whatever this says. specVersion: mintedSpecVersion(), ...queue, ...storage, diff --git a/packages/world-postgres/src/index.ts b/packages/world-postgres/src/index.ts index 0db47b89e2..926d752d3b 100644 --- a/packages/world-postgres/src/index.ts +++ b/packages/world-postgres/src/index.ts @@ -63,9 +63,9 @@ export function createWorld( const streamer = createStreamer(pool, drizzle); return { - // What this world stamps on new runs, which is not the newest version it - // can read: slot identity is readable everywhere and minted only where - // WORKFLOW_SLOT_IDENTITY is set. + // What this world stamps on new runs: slot identity, unless + // WORKFLOW_SLOT_IDENTITY switches it off. Every world reads both schemes + // whatever this says. specVersion: mintedSpecVersion(), ...storage, ...streamer, diff --git a/packages/world-vercel/src/index.ts b/packages/world-vercel/src/index.ts index 42ff155fda..6a68e6845a 100644 --- a/packages/world-vercel/src/index.ts +++ b/packages/world-vercel/src/index.ts @@ -1,5 +1,5 @@ import type { World } from '@workflow/world'; -import { SPEC_VERSION_SUPPORTS_COMPRESSION } from '@workflow/world'; +import { mintedSpecVersion } from '@workflow/world'; import { createAnalytics } from './analytics.js'; import { createRunId, describeRun } from './create-run-id.js'; import { createGetEncryptionKeyForRun } from './encryption.js'; @@ -29,9 +29,12 @@ export function createWorld(config?: APIConfig): World { config?.projectConfig?.projectId || process.env.VERCEL_PROJECT_ID; return { - // Spec v5 adds client-side zstd/gzip payload compression. The server stores - // those payloads opaquely, and v5 remains a superset of v4 attributes. - specVersion: SPEC_VERSION_SUPPORTS_COMPRESSION, + // What this world stamps on new runs: slot identity (spec v6) unless + // WORKFLOW_SLOT_IDENTITY switches it off, in which case v5 — client-side + // zstd/gzip payload compression over a superset of the v4 attributes. + // Either way this world reads both, so the stamp only decides how the runs + // it creates from here on are numbered. + specVersion: mintedSpecVersion(), capabilities: { // workflow-server enforces the `stateUpdatedAt` optimistic-concurrency // guard: creations carrying a stale snapshot are rejected with 412 diff --git a/packages/world/src/spec-version.test.ts b/packages/world/src/spec-version.test.ts index f21596f951..0125e2fb1c 100644 --- a/packages/world/src/spec-version.test.ts +++ b/packages/world/src/spec-version.test.ts @@ -75,31 +75,30 @@ describe('isLegacySpecVersion', () => { }); describe('mintedSpecVersion', () => { - it('mints the current version by default', () => { - expect(mintedSpecVersion({})).toBe(SPEC_VERSION_CURRENT); + it('mints slot identity by default', () => { + expect(mintedSpecVersion({})).toBe(SPEC_VERSION_SLOT_IDENTITY); }); - it('mints slot identity when the flag is set', () => { - for (const value of ['1', 'true']) { + it('mints the previous version when the flag is switched off', () => { + for (const value of ['0', 'false']) { expect(mintedSpecVersion({ [SLOT_IDENTITY_ENV_VAR]: value })).toBe( - SPEC_VERSION_SLOT_IDENTITY + SPEC_VERSION_CURRENT ); } }); - it('treats any other value as off', () => { + it('treats any other value as on', () => { // An unset-but-present variable is the shape a shell leaves behind, and it - // must not silently switch a deployment's event identity scheme. - for (const value of ['', '0', 'false', 'yes']) { + // must not silently switch a deployment's event identity scheme. Opting + // out takes an explicit `0`/`false`. + for (const value of ['', '1', 'true', 'yes']) { expect(mintedSpecVersion({ [SLOT_IDENTITY_ENV_VAR]: value })).toBe( - SPEC_VERSION_CURRENT + SPEC_VERSION_SLOT_IDENTITY ); } }); it('mints nothing a world cannot read', () => { - expect( - requiresNewerWorld(mintedSpecVersion({ [SLOT_IDENTITY_ENV_VAR]: '1' })) - ).toBe(false); + expect(requiresNewerWorld(mintedSpecVersion({}))).toBe(false); }); }); diff --git a/packages/world/src/spec-version.ts b/packages/world/src/spec-version.ts index a86f4f3e90..4b830fd4a8 100644 --- a/packages/world/src/spec-version.ts +++ b/packages/world/src/spec-version.ts @@ -49,10 +49,10 @@ export const SPEC_VERSION_SLOT_IDENTITY = 6 as SpecVersion; * Current spec version (event-sourced architecture with native attributes * and compressed payloads). * - * This is the version new runs are stamped with, which is a *lower* bar than - * the newest version this build can read — see - * {@link SPEC_VERSION_MAX_SUPPORTED}. Slot identity ships behind a flag, so it - * is readable everywhere before it is minted anywhere. + * The floor a world stamps on new runs, and a *lower* bar than the newest + * version this build can read — see {@link SPEC_VERSION_MAX_SUPPORTED}. What a + * world actually stamps comes from {@link mintedSpecVersion}; this is what it + * falls back to when slot identity is switched off. */ export const SPEC_VERSION_CURRENT = SPEC_VERSION_SUPPORTS_COMPRESSION as SpecVersion; @@ -62,16 +62,15 @@ export const SPEC_VERSION_CURRENT = * by {@link requiresNewerWorld} rather than misread. * * Distinct from {@link SPEC_VERSION_CURRENT} because a world has to be able to - * read a version before anything may mint it: the flag that turns slot identity - * on for new runs would otherwise make every world reject the runs it had just - * created. Worlds opt into minting individually, via the `specVersion` they - * declare. + * read a version before anything may mint it, and because a world that mints + * slot identity still has to read the spec-5 runs it created before the switch. + * Worlds opt into minting individually, via the `specVersion` they declare. */ export const SPEC_VERSION_MAX_SUPPORTED = SPEC_VERSION_SLOT_IDENTITY as SpecVersion; /** - * Environment variable that opts new runs into slot identity. + * Environment variable that opts new runs out of slot identity. * * Read per `createWorld()` call rather than at module load, so a test or a * single process can create worlds in both modes. @@ -80,20 +79,20 @@ export const SLOT_IDENTITY_ENV_VAR = 'WORKFLOW_SLOT_IDENTITY'; /** * The spec version a world should stamp on the runs it creates: slot identity - * when {@link SLOT_IDENTITY_ENV_VAR} is set, otherwise + * unless {@link SLOT_IDENTITY_ENV_VAR} disables it, in which case * {@link SPEC_VERSION_CURRENT}. * * Every world reads runs up to {@link SPEC_VERSION_MAX_SUPPORTED} whatever this - * returns, so turning the flag on in one place does not make the runs it creates - * unreadable elsewhere. + * returns, so turning the flag off in one place does not make the runs another + * process created unreadable here. */ export function mintedSpecVersion( env: Record = process.env ): SpecVersion { const value = env[SLOT_IDENTITY_ENV_VAR]; - return value === '1' || value === 'true' - ? SPEC_VERSION_SLOT_IDENTITY - : SPEC_VERSION_CURRENT; + return value === '0' || value === 'false' + ? SPEC_VERSION_CURRENT + : SPEC_VERSION_SLOT_IDENTITY; } /** From 6322aa70b44794776cdca5b498301903e78ce659 Mon Sep 17 00:00:00 2001 From: Peter Wielander Date: Fri, 31 Jul 2026 00:23:56 -0700 Subject: [PATCH 08/35] fix(core): reserve a slot for every event a write publishes MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit A lazy inline step start publishes two events: the World also writes the `step_created` the start deferred. That second event takes the slot immediately below the claim, so the claim has to reserve it — the batch hands out slots synchronously, before any of them land, and a second event numbered off the log as the backend sees it lands on the slot the next start in the batch is holding and costs that start its claim. Co-Authored-By: Claude Opus 5 --- packages/core/src/runtime.ts | 13 ++++++++++- packages/core/src/runtime/helpers.test.ts | 27 +++++++++++++++++++++++ packages/core/src/runtime/helpers.ts | 24 ++++++++++++++++++-- 3 files changed, 61 insertions(+), 3 deletions(-) diff --git a/packages/core/src/runtime.ts b/packages/core/src/runtime.ts index 86a04e0e9c..b877d90e0a 100644 --- a/packages/core/src/runtime.ts +++ b/packages/core/src/runtime.ts @@ -2291,7 +2291,18 @@ export function workflowEntrypoint( // it is assigned in has to be replay-stable. const eventCreateFence = eventCreateFenceFor( inlineClaimLog, - workflowRun.specVersion + workflowRun.specVersion, + { + // A lazy start publishes two events: the World + // writes the step's deferred `step_created` + // alongside the claim, so the batch has to + // reserve a slot for that one too — otherwise + // it lands on the slot the next start in the + // batch is holding and costs that start its + // claim. + extraEvents: + s.lazyStepInput !== undefined ? 1 : 0, + } ); const run = () => executeStep({ diff --git a/packages/core/src/runtime/helpers.test.ts b/packages/core/src/runtime/helpers.test.ts index 15264d404c..3b050f0978 100644 --- a/packages/core/src/runtime/helpers.test.ts +++ b/packages/core/src/runtime/helpers.test.ts @@ -593,6 +593,33 @@ describe('slot bookkeeping', () => { }); }); + it('reserves a slot per extra event and names the top one', () => { + // A lazy inline `step_started` publishes two events: the World also writes + // the `step_created` it deferred, which takes the slot below the claim. + // Reserving it here is what keeps it off the slot the next write of the + // same batch will claim. + const log = toMutableEventLog([slotEvent(1)], null); + expect( + eventCreateFenceFor(log, SPEC_VERSION_SLOT_IDENTITY, { extraEvents: 1 }) + ).toEqual({ eventId: slotEventId(3), maxSlot: 1 }); + expect( + eventCreateFenceFor(log, SPEC_VERSION_SLOT_IDENTITY, { extraEvents: 1 }) + ).toEqual({ eventId: slotEventId(5), maxSlot: 1 }); + // A single-event write in the same batch still gets the next free slot. + expect(eventCreateFenceFor(log, SPEC_VERSION_SLOT_IDENTITY)).toEqual({ + eventId: slotEventId(6), + maxSlot: 1, + }); + }); + + it('burns no slot on an extra event of a ULID-numbered run', () => { + const log = toMutableEventLog([], null); + eventCreateFenceFor(log, SPEC_VERSION_SLOT_IDENTITY - 1, { + extraEvents: 1, + }); + expect(log.reserved).toBe(0); + }); + it('proposes no event id for a ULID-numbered run', () => { // A run whose ids the backend mints must not burn slots either. const log = toMutableEventLog([], null); diff --git a/packages/core/src/runtime/helpers.ts b/packages/core/src/runtime/helpers.ts index 0dda520104..cb5e88f9f6 100644 --- a/packages/core/src/runtime/helpers.ts +++ b/packages/core/src/runtime/helpers.ts @@ -852,13 +852,33 @@ export interface EventCreateFence { * distinct slot per create. `undefined` when the run is fenced neither way, * leaving the create exactly as unfenced as it was before either mechanism * existed. + * + * `extraEvents` is how many events *besides* the one being created this write + * publishes: a lazy inline `step_started` also materializes the `step_created` + * it deferred. Those events take the slots immediately below the claim, so this + * reserves them too and names the top one — a World that writes a pair + * derives the lower id from the one it was given. + * + * The reservation has to happen here rather than at the World or its backend. + * Slots are handed out for a whole concurrent batch synchronously, before any of + * it lands, so a second event numbered off the log as the backend sees it would + * take the slot already promised to the next write in the batch — and every + * write after the first in a fan-out would lose its claim. */ export function eventCreateFenceFor( log: MutableEventLog, - specVersion: number | undefined + specVersion: number | undefined, + options?: { extraEvents?: number } ): EventCreateFence | undefined { if (usesSlotIdentity(specVersion)) { - return { eventId: slotEventId(reserveSlot(log)), maxSlot: log.maxSlot }; + const maxSlot = log.maxSlot; + // The extra events sit below the one being created, matching the order a + // reader expects (a step is created before it starts) — so their slots are + // reserved first and the claim names the last of the run. + for (let i = 0; i < (options?.extraEvents ?? 0); i++) { + reserveSlot(log); + } + return { eventId: slotEventId(reserveSlot(log)), maxSlot }; } const stateUpdatedAt = stateUpdatedAtForCreate(log.events, specVersion); return stateUpdatedAt !== undefined ? { stateUpdatedAt } : undefined; From 3818b89b0da842a1a85d04eb25a0c14a01ed47c6 Mon Sep 17 00:00:00 2001 From: Peter Wielander Date: Thu, 30 Jul 2026 17:46:11 -0700 Subject: [PATCH 09/35] feat(worlds): number events by slot in the Local and Postgres Worlds Slot identity is only useful if a World can actually keep it, so both first-party Worlds now allocate, honour, and defend dense per-run positions: - `SPEC_VERSION_MAX_SUPPORTED` separates the newest version a World can read from the version it stamps, so turning the flag on somewhere does not make the runs it creates unreadable elsewhere. - `mintedSpecVersion()` gives both Worlds one place to opt new runs in. - The Local World allocates under its storage lock and re-probes on a lost exclusive write; the Postgres World makes the events primary key run-scoped and treats a unique violation as "try the next free position", re-probing every round so contention always makes progress. - A caller-claimed position that is already taken is a 409 carrying the events the caller was missing, and nothing is materialized for it. Co-Authored-By: Claude Opus 5 --- .changeset/slot-event-identity-worlds.md | 8 + .../docs/v5/configuration/runtime-tuning.mdx | 9 + packages/core/src/runtime.ts | 113 +++- packages/core/src/runtime/helpers.test.ts | 43 ++ packages/core/src/runtime/helpers.ts | 34 +- packages/core/src/runtime/start.test.ts | 33 +- .../core/src/runtime/world-compatibility.ts | 27 +- packages/core/src/workflow.ts | 26 +- packages/world-local/src/index.ts | 7 +- .../world-local/src/storage/events-storage.ts | 288 ++++++++- packages/world-local/src/storage/helpers.ts | 92 ++- .../src/storage/slot-identity.test.ts | 281 +++++++++ .../world-local/src/storage/slots.test.ts | 241 +++++++ packages/world-local/src/storage/slots.ts | 247 ++++++++ .../0018_run_scoped_event_and_step_keys.sql | 13 + .../src/drizzle/migrations/meta/_journal.json | 7 + packages/world-postgres/src/drizzle/schema.ts | 18 +- packages/world-postgres/src/index.ts | 7 +- packages/world-postgres/src/slots.ts | 225 +++++++ packages/world-postgres/src/storage.ts | 587 ++++++++++++------ .../world-postgres/test/slot-identity.test.ts | 375 +++++++++++ packages/world/src/index.ts | 3 + packages/world/src/slot-identity.test.ts | 25 +- packages/world/src/spec-version.test.ts | 64 +- packages/world/src/spec-version.ts | 52 +- packages/world/src/ulid.ts | 12 + 26 files changed, 2529 insertions(+), 308 deletions(-) create mode 100644 .changeset/slot-event-identity-worlds.md create mode 100644 packages/world-local/src/storage/slot-identity.test.ts create mode 100644 packages/world-local/src/storage/slots.test.ts create mode 100644 packages/world-local/src/storage/slots.ts create mode 100644 packages/world-postgres/src/drizzle/migrations/0018_run_scoped_event_and_step_keys.sql create mode 100644 packages/world-postgres/src/slots.ts create mode 100644 packages/world-postgres/test/slot-identity.test.ts diff --git a/.changeset/slot-event-identity-worlds.md b/.changeset/slot-event-identity-worlds.md new file mode 100644 index 0000000000..863a2f860b --- /dev/null +++ b/.changeset/slot-event-identity-worlds.md @@ -0,0 +1,8 @@ +--- +'@workflow/world': minor +'@workflow/world-local': minor +'@workflow/world-postgres': minor +'@workflow/core': minor +--- + +Number a run's events by position in the Local and Postgres Worlds when `WORKFLOW_SLOT_IDENTITY` is set, so a reader can prove its copy of an event log is complete. diff --git a/docs/content/docs/v5/configuration/runtime-tuning.mdx b/docs/content/docs/v5/configuration/runtime-tuning.mdx index 2f3e68e8c3..110e569d5f 100644 --- a/docs/content/docs/v5/configuration/runtime-tuning.mdx +++ b/docs/content/docs/v5/configuration/runtime-tuning.mdx @@ -48,6 +48,15 @@ For example, a workflow can run a 10-minute inline step even with `WORKFLOW_REPL - Backends that do not support the guard ignore the snapshot; they must not declare the capability, so guard-dependent optimizations stay off against them even when the flag is set. - Set `0` to disable. +### `WORKFLOW_SLOT_IDENTITY` + +- Default: disabled +- Numbers a new run's events by position instead of by ULID: `evnt_…001` is the run's first event, `evnt_…002` its second, with no gaps. A reader can then prove its copy of the log is complete, because the highest number is the event count. +- Contention becomes explicit rather than silent. Two writers proposing the same position cannot both win: the loser gets a 409 ([`SlotConflictError`](/docs/api-reference/workflow-errors/slot-conflict-error)) carrying the events it was missing, and the runtime merges them, replays, and re-proposes at the next free position. +- Applies only to runs created while it is set. A run keeps the identity scheme it was created with for life, so turning the flag on or off never affects runs already in flight. +- Requires a World that supports it. A World that does not rejects the run outright rather than mis-numbering its events. +- Set `1` or `true` to enable. + ## Inline execution ### `WORKFLOW_V2_TIMEOUT_MS` diff --git a/packages/core/src/runtime.ts b/packages/core/src/runtime.ts index b877d90e0a..95d19866a2 100644 --- a/packages/core/src/runtime.ts +++ b/packages/core/src/runtime.ts @@ -4,7 +4,6 @@ import { EntityConflictError, FatalError, MaxEventsExceededError, - PreconditionFailedError, ReplayDivergenceError, RUN_ERROR_CODES, type RunErrorCode, @@ -24,6 +23,7 @@ import { resolveQueueNamespace, SPEC_VERSION_CURRENT, SPEC_VERSION_SUPPORTS_COMPRESSION, + slotFromId, WorkflowInvokePayloadSchema, type WorkflowRun, type World, @@ -57,6 +57,7 @@ import { memoizeEncryptionKey, parseHealthCheckPayload, queueMessage, + requiresFreshReplay, toMutableEventLog, withEventCreateFence, withHealthCheck, @@ -547,6 +548,19 @@ export function workflowEntrypoint( let workflowStartedAt = -1; let preloadedEvents: Event[] | undefined; let preloadedEventsCursor: string | null | undefined; + // Highest slot known to be published on a slot-numbered run, + // for the writes whose snapshot cannot show it: turbo + // backgrounds `run_started` and replays against an empty log, + // so a claim numbered from that log alone would propose a slot + // `run_started` already holds. 0 when the run is not + // slot-numbered — its ids carry no position to compare. + let knownSlotFloor = 0; + const observeSlotFloor = (eventId: string | undefined) => { + const slot = eventId ? slotFromId(eventId) : undefined; + if (slot !== undefined && slot > knownSlotFloor) { + knownSlotFloor = slot; + } + }; // Latency telemetry (TTFS) state — see runtime/step-latency.ts. // Whether this invocation's FIRST event snapshot contained @@ -1021,6 +1035,10 @@ export function workflowEntrypoint( (r) => { const limit = clampMaxEvents(r?.maxEvents); if (limit !== undefined) maxEventsLimit = limit; + // Every write of this invocation is ordered after this + // promise by `runReadyBarrier`, so the slot it reports + // is in hand before the first claim is numbered. + observeSlotFloor(r?.event?.eventId); }, () => {} ); @@ -1088,6 +1106,7 @@ export function workflowEntrypoint( } workflowRun = result.run; maxEventsLimit = clampMaxEvents(result.maxEvents); + observeSlotFloor(result.event?.eventId); // Anchors RSFS — see the declaration above. runStartedReceivedAtMs = Date.now(); @@ -1367,7 +1386,11 @@ export function workflowEntrypoint( // place by the guard's reloads, so a per-iteration // rescan for the slot high-water mark would be wasted // work on an array that never changes identity here. - const waitLog = toMutableEventLog(events, eventsCursor); + const waitLog = toMutableEventLog( + events, + eventsCursor, + knownSlotFloor + ); for (const waitEvent of waitsToComplete) { try { await withEventCreateFence( @@ -1491,6 +1514,20 @@ export function workflowEntrypoint( // point and the inline executeStep mutates eventsCursor. preInlineWriteCursor = eventsCursor; + // One log for everything this replay writes on its way to + // a terminal event: the end-of-run drain inside + // `runWorkflow` (fire-and-forget `*_created` events, and + // the implicit disposal of the abort hooks a completing + // run leaves behind) and the `run_completed` / + // `run_failed` write below. Sharing it is what keeps the + // two from claiming the same slot — the terminal write + // numbers from a snapshot that predates the drain. + const replayWriteLog = toMutableEventLog( + events, + eventsCursor, + knownSlotFloor + ); + // Replay workflow runtimeLogger.debug('Starting workflow replay', { workflowRunId: runId, @@ -1516,7 +1553,8 @@ export function workflowEntrypoint( // `awaitRunReady()` below, so gate those writes on the // backgrounded run_started too. Undefined outside turbo. runReadyBarrier, - world.capabilities + world.capabilities, + replayWriteLog ); await payloadPrewarm; runtimeLogger.debug('Workflow replay completed', { @@ -1527,11 +1565,12 @@ export function workflowEntrypoint( // Workflow completed. Send the snapshot but do NOT // reload-and-retry the create in place: `result` was - // computed by this replay, so a stale (412) rejection must - // force a *fresh replay* (which may observe the new event - // and produce a different result), not re-commit the stale - // result. The catch below lets PreconditionFailedError - // propagate to the queue for re-invocation. + // computed by this replay, so a rejection proving the view + // was incomplete must force a *fresh replay* (which may + // observe the new event and produce a different result), + // not re-commit the stale result. The catch below lets + // `requiresFreshReplay` rejections propagate to the queue + // for re-invocation. try { // Turbo: a workflow that finishes with no steps reaches // here before the backgrounded run_started; order the @@ -1547,7 +1586,7 @@ export function workflowEntrypoint( { requestId, ...eventCreateFenceFor( - toMutableEventLog(events, eventsCursor), + replayWriteLog, workflowRun.specVersion ), } @@ -1637,7 +1676,8 @@ export function workflowEntrypoint( } const suspensionLog = toMutableEventLog( cachedEvents, - eventsCursor + eventsCursor, + knownSlotFloor ); let suspensionResult: Awaited< ReturnType @@ -1653,13 +1693,14 @@ export function workflowEntrypoint( runReadyBarrier, }); } catch (suspensionError) { - // A suspension create whose stale (412) rejection - // survived the in-guard reload retries: schedule an + // A suspension create whose incomplete-view rejection + // (412 stale watermark, or 409 taken slot) survived + // the in-guard reload retries: schedule an // explicit immediate re-invocation (a rethrow relies // on redelivery of a message the turbo path already // acked — the run would stall for the queue's ~300s // default visibility timeout). - if (PreconditionFailedError.is(suspensionError)) { + if (requiresFreshReplay(suspensionError)) { runtimeLogger.warn( 'Suspension event creation rejected as stale after reload retries; re-invoking run for a fresh replay', { workflowRunId: runId, loopIteration } @@ -2273,10 +2314,14 @@ export function workflowEntrypoint( // whole batch so each claim draws its own event slot; // `eventCreateFenceFor` yields undefined for a run // fenced neither way, leaving those claims as they were. - const inlineClaimLog = toMutableEventLog( - cachedEvents ?? [], - eventsCursor - ); + // + // The suspension's own log, not a second one over the + // same snapshot: its reservations are what the hook and + // wait creates just above took, and those events are not + // in `cachedEvents` yet. A fresh log would number these + // claims from the same base and hand the batch's first + // step a slot the suspension already holds. + const inlineClaimLog = suspensionLog; replayBudget.pause(); let stepResults: Awaited< @@ -2416,17 +2461,18 @@ export function workflowEntrypoint( stepExecutionPromises ); } catch (stepErr) { - // A stale (412) rejection of an inline step_started - // claim: the loaded view this batch was scheduled - // from is behind an out-of-band event (e.g. a - // received hook), so the claim was fenced by the + // An incomplete-view rejection of an inline + // step_started claim (412 stale watermark, or 409 + // taken slot): the loaded view this batch was + // scheduled from is behind an out-of-band event (e.g. + // a received hook), so the claim was fenced by the // guard and no step events were written. Abandon the // batch — any optimistic body result is discarded by // executeStep's reconciliation — and re-invoke for a // fresh replay that observes the new event. Wait for // the sibling executions to settle first so no owned // body is in flight when the ack path runs. - if (PreconditionFailedError.is(stepErr)) { + if (requiresFreshReplay(stepErr)) { await Promise.allSettled(stepExecutionPromises); runtimeLogger.warn( 'Inline step claim rejected as stale; re-invoking run for a fresh replay', @@ -2623,17 +2669,18 @@ export function workflowEntrypoint( } } } else { - // Stale-snapshot rejection of a result-bearing create - // (run_completed sends the snapshot but is intentionally - // NOT retried in place), or one that survived the - // in-guard reload retries. Don't fail the run — schedule - // an explicit immediate re-invocation so a fresh replay - // observes the new event. Rethrowing instead would rely - // on redelivery of the CURRENT message, which the turbo - // path has already acked — empirically the run then - // stalls for the queue's ~300s default visibility - // timeout before completing. - if (PreconditionFailedError.is(err)) { + // Incomplete-view rejection of a result-bearing create — + // a stale watermark (412) or a taken slot (409), either + // on `run_completed` (which sends its fence but is + // intentionally NOT retried in place) or on a create + // that survived the in-guard reload retries. Don't fail + // the run — schedule an explicit immediate re-invocation + // so a fresh replay observes the new event. Rethrowing + // instead would rely on redelivery of the CURRENT + // message, which the turbo path has already acked — + // empirically the run then stalls for the queue's ~300s + // default visibility timeout before completing. + if (requiresFreshReplay(err)) { runtimeLogger.warn( 'Event creation rejected as stale; re-invoking run for a fresh replay', { workflowRunId: runId, loopIteration } diff --git a/packages/core/src/runtime/helpers.test.ts b/packages/core/src/runtime/helpers.test.ts index 3b050f0978..ca1a3ff4b5 100644 --- a/packages/core/src/runtime/helpers.test.ts +++ b/packages/core/src/runtime/helpers.test.ts @@ -1,4 +1,5 @@ import { + EntityConflictError, PreconditionFailedError, SlotConflictError, WorkflowWorldError, @@ -29,6 +30,7 @@ import { memoizeEncryptionKey, mergeLoadedEvents, PRECONDITION_MAX_RELOAD_RETRIES, + requiresFreshReplay, reserveSlot, stateUpdatedAtForCreate, toMutableEventLog, @@ -537,6 +539,26 @@ describe('slot bookkeeping', () => { ).toBe(0); }); + it('starts at a floor the snapshot cannot show', () => { + // Turbo replays against an empty log while its `run_started` write is still + // in flight, so the snapshot alone would number the first claim onto a slot + // that write already holds. + const log = toMutableEventLog([], null, 2); + expect(log.maxSlot).toBe(2); + expect(reserveSlot(log)).toBe(3); + }); + + it('ignores a floor the snapshot has already passed', () => { + const log = toMutableEventLog([slotEvent(5)], 'c0', 2); + expect(log.maxSlot).toBe(5); + }); + + it('keeps the floor across a merge', () => { + const log = toMutableEventLog([], null, 2); + mergeLoadedEvents(log, [slotEvent(1)]); + expect(log.maxSlot).toBe(2); + }); + it('never lowers maxSlot when an older delta is merged in', () => { const log = toMutableEventLog([slotEvent(1), slotEvent(3)], 'c0'); mergeLoadedEvents(log, [slotEvent(2)]); @@ -837,6 +859,27 @@ describe('withEventCreateFence', () => { }); }); +describe('requiresFreshReplay', () => { + it('covers both fences, so neither numbering fails the run', () => { + // Each fence reports an incomplete view in its own dialect. A caller that + // recognises only one of them fails runs on the other. + expect(requiresFreshReplay(new PreconditionFailedError('stale'))).toBe( + true + ); + expect( + requiresFreshReplay( + new SlotConflictError('taken', { eventId: slotEventId(3) }) + ) + ).toBe(true); + }); + + it('leaves every other rejection to its own handler', () => { + expect(requiresFreshReplay(new EntityConflictError('exists'))).toBe(false); + expect(requiresFreshReplay(new Error('boom'))).toBe(false); + expect(requiresFreshReplay(undefined)).toBe(false); + }); +}); + describe('withPreconditionRetry', () => { let originalGuard: string | undefined; diff --git a/packages/core/src/runtime/helpers.ts b/packages/core/src/runtime/helpers.ts index cb5e88f9f6..407f202635 100644 --- a/packages/core/src/runtime/helpers.ts +++ b/packages/core/src/runtime/helpers.ts @@ -669,12 +669,26 @@ export interface MutableEventLog { reserved: number; } -/** A `MutableEventLog` over a freshly loaded snapshot. */ +/** + * A `MutableEventLog` over a freshly loaded snapshot. + * + * `slotFloor` is a slot known to be published that the snapshot may not contain + * — the run's own `run_started`, whose write turbo backgrounds while replaying + * against an empty log. Numbering a claim from the snapshot alone would then + * propose a slot that is already taken, so every first write of a turbo + * invocation would conflict and cost the run an extra replay. + */ export function toMutableEventLog( events: Event[], - cursor: string | null + cursor: string | null, + slotFloor = 0 ): MutableEventLog { - return { events, cursor, maxSlot: maxSlotOf(events), reserved: 0 }; + return { + events, + cursor, + maxSlot: Math.max(maxSlotOf(events), slotFloor), + reserved: 0, + }; } /** @@ -982,6 +996,20 @@ export function withEventCreateFence( ); } +/** + * Whether a rejected event create means "this replay's view of the log was + * incomplete", the one condition whose only remedy is replaying from the top. + * + * Both fences report it, one per numbering: a 412 says the snapshot's watermark + * is behind, a 409 says the slot this replay counted to is already occupied. + * Neither is a failure of the run — the run's own decisions may simply need + * revising against the events it did not see — so a caller that gets one + * re-invokes for a fresh replay rather than failing. + */ +export function requiresFreshReplay(error: unknown): boolean { + return PreconditionFailedError.is(error) || SlotConflictError.is(error); +} + /** * CORS headers for health check responses. * Allows the observability UI to check endpoint health from a different origin. diff --git a/packages/core/src/runtime/start.test.ts b/packages/core/src/runtime/start.test.ts index 29a7c80419..d04a415bb7 100644 --- a/packages/core/src/runtime/start.test.ts +++ b/packages/core/src/runtime/start.test.ts @@ -2,6 +2,7 @@ import { WorkflowRuntimeError, WorkflowWorldError } from '@workflow/errors'; import { SPEC_VERSION_CURRENT, SPEC_VERSION_LEGACY, + SPEC_VERSION_MAX_SUPPORTED, SPEC_VERSION_SUPPORTS_ATTRIBUTES, SPEC_VERSION_SUPPORTS_CBOR_QUEUE_TRANSPORT, } from '@workflow/world'; @@ -136,7 +137,7 @@ describe('start', () => { } as any); await expect(start(validWorkflow, [])).rejects.toThrow( - 'requires a World with matching spec version' + 'requires a World with spec version' ); expect(mockEventsCreate).not.toHaveBeenCalled(); expect(mockQueue).not.toHaveBeenCalled(); @@ -174,7 +175,7 @@ describe('start', () => { } as any); await expect(start(validWorkflow, [])).rejects.toThrow( - 'requires a World with matching spec version' + 'requires a World with spec version' ); expect(mockEventsCreate).not.toHaveBeenCalled(); expect(mockQueue).not.toHaveBeenCalled(); @@ -186,19 +187,43 @@ describe('start', () => { }); setWorld({ - specVersion: SPEC_VERSION_CURRENT + 1, + specVersion: SPEC_VERSION_MAX_SUPPORTED + 1, getDeploymentId: vi.fn().mockResolvedValue('deploy_123'), events: { create: mockEventsCreate }, queue: mockQueue, } as any); await expect(start(validWorkflow, [])).rejects.toThrow( - 'requires a World with matching spec version' + 'requires a World with spec version' ); expect(mockEventsCreate).not.toHaveBeenCalled(); expect(mockQueue).not.toHaveBeenCalled(); }); + it('accepts a world that mints a newer version this runtime supports', async () => { + // A world opted into slot identity stamps a version above the runtime's + // current one. The runtime can read and write those runs, so the + // handshake has to pass and the run has to keep the world's version. + const validWorkflow = Object.assign(() => Promise.resolve('result'), { + workflowId: 'test-workflow', + }); + + setWorld({ + specVersion: SPEC_VERSION_MAX_SUPPORTED, + getDeploymentId: vi.fn().mockResolvedValue('deploy_123'), + events: { create: mockEventsCreate }, + queue: mockQueue, + } as any); + + await start(validWorkflow, []); + + expect(mockEventsCreate).toHaveBeenCalledWith( + expect.stringMatching(/^wrun_/), + expect.objectContaining({ specVersion: SPEC_VERSION_MAX_SUPPORTED }), + expect.anything() + ); + }); + it('should use provided specVersion when passed in options', async () => { const validWorkflow = Object.assign(() => Promise.resolve('result'), { workflowId: 'test-workflow', diff --git a/packages/core/src/runtime/world-compatibility.ts b/packages/core/src/runtime/world-compatibility.ts index c06c26eefb..f3de904359 100644 --- a/packages/core/src/runtime/world-compatibility.ts +++ b/packages/core/src/runtime/world-compatibility.ts @@ -1,19 +1,40 @@ import { WorkflowRuntimeError } from '@workflow/errors'; import type { World } from '@workflow/world'; -import { SPEC_VERSION_CURRENT } from '@workflow/world'; +import { + SPEC_VERSION_CURRENT, + SPEC_VERSION_MAX_SUPPORTED, +} from '@workflow/world'; type WorldSpecVersionMetadata = Pick; +/** + * Rejects a World whose protocol this runtime does not speak. + * + * A World declares the spec version it stamps on the runs it creates. Anything + * from {@link SPEC_VERSION_CURRENT} up to {@link SPEC_VERSION_MAX_SUPPORTED} is + * fine: the upper end covers a World opted into a newer identity scheme that + * this runtime already understands, and only versions this runtime has no code + * for are refused. Below the current version means the World package predates + * this runtime and cannot record what it emits. + */ export function assertWorldSupportsRuntimeProtocol( world: WorldSpecVersionMetadata ): void { - if (world.specVersion === SPEC_VERSION_CURRENT) { + if ( + world.specVersion !== undefined && + world.specVersion >= SPEC_VERSION_CURRENT && + world.specVersion <= SPEC_VERSION_MAX_SUPPORTED + ) { return; } const supportedVersion = world.specVersion ?? 'none'; + const supported = + SPEC_VERSION_CURRENT === SPEC_VERSION_MAX_SUPPORTED + ? `${SPEC_VERSION_CURRENT}` + : `${SPEC_VERSION_CURRENT} to ${SPEC_VERSION_MAX_SUPPORTED}`; throw new WorkflowRuntimeError( - `This Workflow runtime requires a World with matching spec version ${SPEC_VERSION_CURRENT}, ` + + `This Workflow runtime requires a World with spec version ${supported}, ` + `but the configured World declares spec version ${supportedVersion}. ` + 'Install a World package version compatible with the current Workflow runtime.' ); diff --git a/packages/core/src/workflow.ts b/packages/core/src/workflow.ts index fde4b70250..97c9acee99 100644 --- a/packages/core/src/workflow.ts +++ b/packages/core/src/workflow.ts @@ -18,6 +18,7 @@ import { runtimeLogger } from './logger.js'; import type { WorkflowOrchestratorContext } from './private.js'; import { ReplayPayloadCache } from './replay-payload-cache.js'; import { getPortLazy } from './runtime/get-port-lazy.js'; +import type { MutableEventLog } from './runtime/helpers.js'; import { runIdCreatedAt } from './runtime/run-id-time.js'; import { handleSuspension } from './runtime/suspension-handler.js'; import { getWorld } from './runtime/world.js'; @@ -74,7 +75,15 @@ async function drainPendingQueueItems( * In turbo mode, gates final `*_created` writes on backgrounded * `run_started`. Undefined when `run_started` is awaited. */ - runReadyBarrier?: Promise + runReadyBarrier?: Promise, + /** + * The replay's event log, so the drain's writes claim their slots from the + * same source the terminal `run_completed` / `run_failed` write draws from. + * Without it the drain writes unfenced — the World picks the next free slot — + * and the terminal write, numbering from a snapshot taken before the drain, + * proposes the slot the drain just took and loses it. + */ + eventLog?: MutableEventLog ): Promise { if (pendingQueue.size === 0) return; // Implicitly dispose any abort hooks (system hooks) that are still alive at @@ -101,6 +110,7 @@ async function drainPendingQueueItems( world, run: workflowRun, runReadyBarrier, + eventLog, }); } catch (err) { runtimeLogger.warn( @@ -137,7 +147,13 @@ export async function runWorkflow( * Features supported by the World executing this workflow. Missing * capabilities are treated as unsupported. */ - worldCapabilities?: WorldCapabilities + worldCapabilities?: WorldCapabilities, + /** + * The caller's event log for this replay. Its only use here is the end-of-run + * drain, whose writes have to be ordered with the caller's terminal write — + * see {@link drainPendingQueueItems}. + */ + eventLog?: MutableEventLog ): Promise { return trace(`workflow.run ${workflowRun.workflowName}`, async (span) => { span?.setAttributes({ @@ -859,7 +875,8 @@ export async function runWorkflow( vmGlobalThis, workflowRun, 'completed', - runReadyBarrier + runReadyBarrier, + eventLog ); return dehydrated; @@ -876,7 +893,8 @@ export async function runWorkflow( vmGlobalThis, workflowRun, 'failed', - runReadyBarrier + runReadyBarrier, + eventLog ); throw err; diff --git a/packages/world-local/src/index.ts b/packages/world-local/src/index.ts index f8e2166fb7..2014a75652 100644 --- a/packages/world-local/src/index.ts +++ b/packages/world-local/src/index.ts @@ -2,7 +2,7 @@ import { promises as fs } from 'node:fs'; import { rm } from 'node:fs/promises'; import path from 'node:path'; import type { QueuePrefix, World } from '@workflow/world'; -import { reenqueueActiveRuns, SPEC_VERSION_CURRENT } from '@workflow/world'; +import { mintedSpecVersion, reenqueueActiveRuns } from '@workflow/world'; import { warnIfRunningInVercelDeployment } from './build-target-mismatch.js'; import type { Config } from './config.js'; import { config, resolveRecoverActiveRuns } from './config.js'; @@ -72,7 +72,10 @@ export function createWorld(args?: Partial): LocalWorld { ); const recoverActiveRuns = resolveRecoverActiveRuns(mergedConfig); return { - specVersion: SPEC_VERSION_CURRENT, + // What this world stamps on new runs, which is not the newest version it + // can read: slot identity is readable everywhere and minted only where + // WORKFLOW_SLOT_IDENTITY is set. + specVersion: mintedSpecVersion(), ...queue, ...storage, ...instrumentObject('world.streams', { diff --git a/packages/world-local/src/storage/events-storage.ts b/packages/world-local/src/storage/events-storage.ts index 7efd49cb1e..98e705e98d 100644 --- a/packages/world-local/src/storage/events-storage.ts +++ b/packages/world-local/src/storage/events-storage.ts @@ -5,11 +5,13 @@ import { HookNotFoundError, RunExpiredError, RunNotSupportedError, + SlotConflictError, TooEarlyError, WorkflowRunNotFoundError, WorkflowWorldError, } from '@workflow/errors'; import type { + CreateEventParams, Event, EventResult, Hook, @@ -34,8 +36,12 @@ import { isTerminalWorkflowRunStatus, requiresNewerWorld, SPEC_VERSION_CURRENT, + SPEC_VERSION_MAX_SUPPORTED, StepSchema, + slotEventId, + slotFromId, ulidToDate, + usesSlotIdentity, validateAttributeChanges, validateUlidTimestamp, WaitSchema, @@ -85,6 +91,7 @@ import { } from './hooks-storage.js'; import { handleLegacyEvent } from './legacy.js'; import { withRunFileLock } from './runs-storage.js'; +import { createSlotBook, RUN_CREATED_SLOT } from './slots.js'; /** * Per-run event ceiling the Local World reports on run responses (mirrors the @@ -506,6 +513,8 @@ export function createEventsStorage( const cachedPathsByRunId = new Map>(); let totalCachedEventBytes = 0; + const slots = createSlotBook(basedir, tag); + function deleteCachedEvent(eventPath: string): void { const event = eventCache.get(eventPath); if (!event) { @@ -525,6 +534,7 @@ export function createEventsStorage( for (const cachedPath of cachedPathsByRunId.get(runId) ?? []) { deleteCachedEvent(cachedPath); } + slots.forget(runId); } function clearCache(): void { @@ -532,6 +542,7 @@ export function createEventsStorage( cachedEventBytes.clear(); cachedPathsByRunId.clear(); totalCachedEventBytes = 0; + slots.clear(); } function cacheEvent( @@ -592,6 +603,64 @@ export function createEventsStorage( } } + /** + * The events a caller that just lost a slot is missing: one ascending page of + * the run's log after the cursor it wrote from, minus anything at or below the + * highest slot it already held. Because slots are dense, that second filter is + * exact — a caller cannot be missing an event whose position it can name. + * + * Returned inline with the conflict so the common case (a handful of events + * arrived out of band) costs the caller no extra round-trip. `hasMore` is + * forwarded verbatim: an overflowing delta is the caller's signal to page from + * `cursor` instead of treating this as the whole story. + */ + async function eventsAfterClaim( + runId: string, + params: CreateEventParams | undefined + ): Promise<{ events: Event[]; cursor: string | null; hasMore: boolean }> { + const page = await paginatedFileSystemQuery({ + directory: path.join(basedir, 'events'), + schema: EventSchema, + cachedItems: eventCache, + filePrefix: `${runId}-`, + sortOrder: 'asc', + ...(typeof params?.sinceCursor === 'string' + ? { cursor: params.sinceCursor } + : {}), + getCreatedAt: getObjectCreatedAt('evnt'), + getId: (event) => event.eventId, + }); + const maxSlot = params?.maxSlot ?? 0; + const resolveData = params?.resolveData ?? DEFAULT_RESOLVE_DATA_OPTION; + const missing = page.data.filter( + (event) => (slotFromId(event.eventId) ?? 0) > maxSlot + ); + return { + events: + resolveData === 'none' + ? missing.map((event) => stripEventDataRefs(event, resolveData)) + : missing, + cursor: page.cursor, + hasMore: page.hasMore, + }; + } + + /** + * The 409 a caller gets when the slot it named turns out to belong to someone + * else, carrying the events it is missing so it can merge, replay and + * re-propose at a free position. + */ + async function slotConflict( + runId: string, + eventId: string, + params: CreateEventParams | undefined + ): Promise { + return new SlotConflictError( + `Slot ${slotFromId(eventId)} of run "${runId}" is already taken`, + { eventId, ...(await eventsAfterClaim(runId, params)) } + ); + } + async function storeEvent(event: Event): Promise { const eventPath = taggedPath( basedir, @@ -643,6 +712,36 @@ export function createEventsStorage( if ('correlationId' in data && typeof data.correlationId === 'string') { assertSafeEntityId('correlationId', data.correlationId); } + if (params?.eventId !== undefined) { + assertSafeEntityId('eventId', params.eventId); + } + + // A slot-numbered create reserves its position before running the + // validation and materialization that may still reject it. Handing the + // reservation back on the way out is what keeps the log dense: an + // abandoned slot below a sibling's published one is a hole that can never + // be filled, and a log with a hole can no longer prove it is complete. + const reserved = new Set(); + let reservedRunId: string | undefined; + /** + * Hands the slots of a create that never published back to the allocator, + * so an abandoned reservation below a sibling's published slot does not + * become a hole the run can never fill. + */ + async function releasingSlots( + result: Promise + ): Promise { + try { + return await result; + } catch (error) { + if (reservedRunId !== undefined) { + for (const slot of reserved) { + slots.release(reservedRunId, slot); + } + } + throw error; + } + } // Step lifecycle events are serialized per-step via an in-process mutex // so that the "check state, then write" sequence in step_started / @@ -653,7 +752,9 @@ export function createEventsStorage( const lockKey = tag ? `${runId}-${data.correlationId}.${tag}` : `${runId}-${data.correlationId}`; - return withInProcessLock(stepLocks, lockKey, () => createImpl()); + return releasingSlots( + withInProcessLock(stepLocks, lockKey, () => createImpl()) + ); } // `hook_created` is serialized per-(runId, hookId) so the // "claim token, write hook entity, write event" sequence runs to @@ -682,9 +783,11 @@ export function createEventsStorage( const lockKey = tag ? `${runId}-${data.correlationId}.hook.${tag}` : `${runId}-${data.correlationId}.hook`; - return withInProcessLock(hookLocks, lockKey, () => createImpl()); + return releasingSlots( + withInProcessLock(hookLocks, lockKey, () => createImpl()) + ); } - return createImpl(); + return releasingSlots(createImpl()); async function createImpl(): Promise { // Most paths use the freshly-generated candidate eventId. The @@ -719,6 +822,18 @@ export function createEventsStorage( // specVersion is always sent by the runtime, but we provide a fallback for safety const effectiveSpecVersion = data.specVersion ?? SPEC_VERSION_CURRENT; + // Whether this run numbers its events by slot. Read from what was + // persisted, never from this request or this build, so a run stays in + // the mode it was created in for life — a run whose log holds ULID ids + // must never be handed a slot id, and vice versa. `run_created` is the + // one event that decides the mode instead of reading it; the + // resilient-start path below decides it too, on the request that + // creates the run. + let slotMode = + data.eventType === 'run_created' + ? usesSlotIdentity(effectiveSpecVersion) + : await slots.usesSlots(effectiveRunId); + // Get current run state for validation (if not creating a new run) // Skip run validation for step_completed and step_retrying - they only operate // on running steps, and running steps are always allowed to modify regardless @@ -800,8 +915,14 @@ export function createEventsStorage( ); if (created) { - // We created the run — also write the run_created event. - const runCreatedEventId = `evnt_${monotonicUlid()}`; + // We created the run, so this request also decided its mode. + slotMode = usesSlotIdentity(effectiveSpecVersion); + // We created the run — also write the run_created event. Its + // slot needs no allocation: a run's own `run_created` provably + // has nothing before it. + const runCreatedEventId = slotMode + ? slotEventId(RUN_CREATED_SLOT) + : `evnt_${monotonicUlid()}`; const runCreatedEvent: Event = { eventType: 'run_created', runId: effectiveRunId, @@ -820,6 +941,7 @@ export function createEventsStorage( }, }; await storeEvent(runCreatedEvent); + slots.observe(effectiveRunId, runCreatedEventId); currentRun = createdRun; } else { // Run already exists (concurrent run_created won the @@ -836,6 +958,17 @@ export function createEventsStorage( } } + // The run entity we just read is the authority on the mode, and it can + // appear between the probe above and this read: start() issues + // `run_created` and the queue send concurrently, so the delivery's + // `run_started` can arrive while the run is still being published. A + // stale "no" there would number that one event with a ULID on an + // otherwise slot-numbered run, and the hole it leaves in the numbering + // costs the log its completeness proof for life. + if (currentRun && data.eventType !== 'run_created') { + slotMode = usesSlotIdentity(currentRun.specVersion); + } + // run_failed on a non-existent run is rejected to match the // postgres and vercel worlds, which both surface this as a // WorkflowRunNotFoundError rather than silently persisting an @@ -857,7 +990,7 @@ export function createEventsStorage( if (requiresNewerWorld(currentRun.specVersion)) { throw new RunNotSupportedError( currentRun.specVersion!, - SPEC_VERSION_CURRENT + SPEC_VERSION_MAX_SUPPORTED ); } @@ -873,6 +1006,55 @@ export function createEventsStorage( } } + // ============================================================ + // EVENT ID: the caller's slot claim, an allocated slot, or a ULID + // ============================================================ + // A slot-numbered run's ids name positions in its log, so an id is + // either claimed by a caller that holds the log (and is therefore + // asserting the log is complete up to that position) or allocated here + // for a caller that has no log — a step completion reporting in, a + // cancellation from an API call. + if (params?.eventId !== undefined) { + const claimedSlot = slotFromId(params.eventId); + if (!slotMode) { + throw new WorkflowWorldError( + `Event id "${params.eventId}" was supplied for run "${effectiveRunId}", whose events are not numbered by slot`, + { status: 400 } + ); + } + if (claimedSlot === undefined) { + throw new WorkflowWorldError( + `Event id "${params.eventId}" is not a slot id, and run "${effectiveRunId}" numbers its events by slot`, + { status: 400 } + ); + } + eventId = params.eventId; + reservedRunId = effectiveRunId; + reserved.add(claimedSlot); + slots.claim(effectiveRunId, claimedSlot); + if (await slots.isWritten(effectiveRunId, claimedSlot)) { + // Reject a doomed claim before the materialization below creates + // the step, hook or wait this event will now never accompany. A + // caller that re-proposes at the next slot would otherwise + // collide with its own orphan and read that as "my write already + // landed". See SlotBook.isWritten. + throw await slotConflict(effectiveRunId, eventId, params); + } + } else if (slotMode) { + reservedRunId = effectiveRunId; + // A run's own `run_created` owns the first slot — provably, since + // nothing precedes it — so every other event allocates above it, even + // when that event is the first to arrive here. + const slot = await slots.reserve( + effectiveRunId, + data.eventType === 'run_created' + ? RUN_CREATED_SLOT + : RUN_CREATED_SLOT + 1 + ); + reserved.add(slot); + eventId = slotEventId(slot); + } + // ============================================================ // VALIDATION: Terminal state and event ordering checks // ============================================================ @@ -1127,13 +1309,40 @@ export function createEventsStorage( // strictly dominates all visible events of the run guarantees the // terminal event replays last. See mintRunDominantEventKey for // the dominance argument. - const dominantKey = await mintRunDominantEventKey( - basedir, - effectiveRunId, - tag - ); - eventId = dominantKey.eventId; - event = { ...event, eventId, createdAt: dominantKey.createdAt }; + // + // A key the *caller* chose is left alone. Its slot was picked from + // the caller's own log, so a concurrent event either sits below it + // (and already replays first) or takes the slot itself — in which + // case the publish below conflicts and the caller merges and + // re-proposes, which is the stronger answer. Re-numbering it here + // would also be actively wrong: the caller reserves slots for a + // whole flush of concurrent ops at once, so moving this one to + // "highest visible + 1" would steal the slot a sibling op is still + // in flight with. + if (params?.eventId === undefined) { + const dominantKey = await mintRunDominantEventKey( + basedir, + effectiveRunId, + tag, + slotMode + ); + const staleSlot = slotFromId(eventId); + const dominantSlot = slotFromId(dominantKey.eventId); + if (staleSlot !== undefined && staleSlot !== dominantSlot) { + // Only reachable when the log moved under us, which means the + // slot we held is now someone else's written event — handing it + // back leaves no hole. + reserved.delete(staleSlot); + slots.release(effectiveRunId, staleSlot); + } + if (dominantSlot !== undefined) { + reservedRunId = effectiveRunId; + reserved.add(dominantSlot); + slots.claim(effectiveRunId, dominantSlot); + } + eventId = dominantKey.eventId; + event = { ...event, eventId, createdAt: dominantKey.createdAt }; + } } // Create/update entity based on event type (event-sourced architecture) @@ -1505,13 +1714,19 @@ export function createEventsStorage( ); // Write the synthetic step_created event so replay observes it // (the client step consumer sets hasCreatedEvent only on a - // step_created event). Its eventId is a fresh monotonic ULID. + // step_created event). Its eventId is a second slot, or a fresh + // monotonic ULID — one request, two events. // Ordering vs. the step_started event row does not affect // correctness: the step_started consumer is a no-op and only // step_created flips hasCreatedEvent, so the end state is the // same whichever sorts first — this matches the resilient // run_started → run_created precedent in this file. - const stepCreatedEventId = `evnt_${monotonicUlid()}`; + let stepCreatedEventId = `evnt_${monotonicUlid()}`; + if (slotMode) { + const slot = await slots.reserve(effectiveRunId); + reserved.add(slot); + stepCreatedEventId = slotEventId(slot); + } const stepCreatedEvent: Event = { eventType: 'step_created', runId: effectiveRunId, @@ -1533,6 +1748,7 @@ export function createEventsStorage( ), stepCreatedEvent ); + slots.observe(effectiveRunId, stepCreatedEventId); validatedStep = createdStep; stepCreatedLazily = true; } @@ -2215,11 +2431,17 @@ export function createEventsStorage( // race here; whoever links the file first wins, the loser // throws EntityConflictError, and the runtime's existing // concurrent-replay catch path at suspension-handler.ts:142 - // swallows it. For all other event types, eventIds are - // monotonic ULIDs (globally unique by construction) so a - // collision indicates a real bug and EntityConflictError is + // swallows it. For all other event types of a ULID-numbered run, + // eventIds are monotonic ULIDs (globally unique by construction) so + // a collision indicates a real bug and EntityConflictError is // also the right surface — same shape as step_created's // claim-file behavior. + // + // A slot-numbered run collides by design: the id names a position in + // the log, so a loser is not a bug but a writer whose log was missing + // an event. It gets a SlotConflictError carrying that event instead + // (see below), and this write is the authority that decides it — the + // allocator's book is only ever a hint. // Last-instant re-validation for `hook_received` (see the acceptance // check above). The per-hook in-process lock already serializes // resume vs. dispose within one storage instance; this second check @@ -2311,10 +2533,15 @@ export function createEventsStorage( ); const staged = await writeExclusive(stagedPath, serializedEvent); if (!staged) { - // eventId is a freshly generated ULID; its staging path can - // only be occupied by a previous crashed attempt of this very - // event, which never promoted. Surface the same conflict shape - // as a visible-path collision. + // For a ULID-numbered run the eventId is freshly generated, so + // its staging path can only be occupied by a previous crashed + // attempt of this very event, which never promoted. A + // slot-numbered run can also collide here with another instance + // that allocated the same slot from its own book. Either way the + // event is not reader-visible, so there is no delta to hand back + // and nothing for the caller to merge: surface the same conflict + // shape as a visible-path collision, and let the allocator + // re-probe on the retry. throw new EntityConflictError( `Event "${eventId}" already exists for run "${effectiveRunId}"` ); @@ -2364,6 +2591,22 @@ export function createEventsStorage( tag ); } + if (slotMode) { + // Losing a slot means someone else's event occupies this position, + // so the log this event was derived from is missing at least that + // event — the whole proposed event is stale, not just its id. Hand + // back what the caller is missing so it can merge, replay and + // re-propose, and forget the run's book so the next allocation + // re-reads the log this instance evidently does not have. + // + // Reaching here means the slot was taken *after* the pre-check at + // the claim site, so the entity this event was going to describe + // has already been materialized. Only two storage instances + // sharing a directory can do that, since one instance's book + // hands the same slot to nobody else. + slots.forget(effectiveRunId); + throw await slotConflict(effectiveRunId, eventId, params); + } throw new EntityConflictError( `Event "${eventId}" already exists for run "${effectiveRunId}"` ); @@ -2372,6 +2615,7 @@ export function createEventsStorage( // The event is now committed; cache it so an immediate sequential // replay can serve it without rereading from disk. rememberStoredEvent(event, eventPath, serializedEvent); + slots.observe(effectiveRunId, eventId); // Write the hook entity ONLY now that the event publish has // committed. Doing this earlier (in the `hook_created` diff --git a/packages/world-local/src/storage/helpers.ts b/packages/world-local/src/storage/helpers.ts index d7c2cc78bb..2817fb0574 100644 --- a/packages/world-local/src/storage/helpers.ts +++ b/packages/world-local/src/storage/helpers.ts @@ -1,6 +1,7 @@ import { createHash } from 'node:crypto'; import fs from 'node:fs/promises'; import path from 'node:path'; +import { FIRST_SLOT, maxSlotOf, slotEventId } from '@workflow/world'; import { decodeTime, monotonicFactory } from 'ulid'; import { hasTag, @@ -208,6 +209,46 @@ export async function reapPendingHookEvents( } } +/** + * The event ids of `runId` that are visible in the given tag's view, read from + * the event filenames alone — no file contents, so the cost is one `readdir` + * however large the log is. + * + * A missing `events` directory means the run provably has no events yet. Any + * other failure is thrown: callers derive an event key from this scan, and a + * silently short answer would mint a key that collides with, or fails to + * dominate, an event that is actually there. + */ +export async function listRunEventIds( + basedir: string, + runId: string, + tag?: string +): Promise { + let files: string[] = []; + try { + files = await fs.readdir(path.join(basedir, 'events')); + } catch (error) { + if ((error as NodeJS.ErrnoException).code !== 'ENOENT') { + throw error; + } + } + const prefix = `${runId}-`; + const eventIds: string[] = []; + for (const file of files) { + if (!file.startsWith(prefix) || !file.endsWith('.json')) { + continue; + } + const fileId = file.slice(0, -'.json'.length); + // Mirror read visibility: untagged files are visible to every tag, + // tagged files only to their own tag. + if (!isUntagged(fileId) && !(tag && hasTag(fileId, tag))) { + continue; + } + eventIds.push(stripTag(fileId).slice(prefix.length)); + } + return eventIds; +} + /** * Mint an event key (eventId + createdAt) that sorts strictly AFTER every * reader-visible event of the run in the given tag's view. @@ -229,38 +270,35 @@ export async function reapPendingHookEvents( * >= every visible event's `createdAt`, which was stamped at that event's * `createImpl()` entry — before its publish, and thus before this call. * Equal-`createdAt` ties fall to the strictly-dominant eventId. + * + * A slot-numbered run takes the slot above the highest visible one, which + * dominates by construction, paired with the wall clock — `createdAt` needs + * only to be >= every visible one, by the same argument as above. This is + * the one allocation that deliberately does *not* fill a hole below the max: + * a lower slot would sort before the events it has to follow, and density + * matters less here than replay order, since a hole below a terminal event + * means the run already lost an event it can never write. */ export async function mintRunDominantEventKey( basedir: string, runId: string, - tag?: string + tag: string | undefined, + slotMode: boolean ): Promise<{ eventId: string; createdAt: Date }> { - let files: string[] = []; - try { - files = await fs.readdir(path.join(basedir, 'events')); - } catch (error) { - // Only ENOENT ("no events directory yet") means there is provably - // nothing visible to dominate. Any other failure would silently mint a - // wall-clock key with no dominance guarantee over an already-accepted - // hook — abort the terminal transition instead; its retry re-runs this - // scan. - if ((error as NodeJS.ErrnoException).code !== 'ENOENT') { - throw error; - } + const eventIds = await listRunEventIds(basedir, runId, tag); + if (slotMode) { + // Above every event on disk, and above the run's own first slot even when + // that event has not landed yet: only `run_created` may occupy it, and a + // terminal event is never the run's first. + return { + eventId: slotEventId( + Math.max(maxSlotOf(eventIds.map(toEventRef)), FIRST_SLOT) + 1 + ), + createdAt: new Date(), + }; } - const prefix = `${runId}-`; let maxUlid: string | null = null; - for (const file of files) { - if (!file.startsWith(prefix) || !file.endsWith('.json')) { - continue; - } - const fileId = file.slice(0, -'.json'.length); - // Mirror read visibility: untagged files are visible to every tag, - // tagged files only to their own tag. - if (!isUntagged(fileId) && !(tag && hasTag(fileId, tag))) { - continue; - } - const candidate = stripTag(fileId).slice(prefix.length); + for (const candidate of eventIds) { if (!maxUlid || candidate > maxUlid) { maxUlid = candidate; } @@ -279,6 +317,10 @@ export async function mintRunDominantEventKey( return { eventId: `evnt_${monotonicUlid(ts)}`, createdAt: new Date(ts) }; } +function toEventRef(eventId: string): { eventId: string } { + return { eventId }; +} + /** * Path of the exclusive-create claim file that reserves a hook token. */ diff --git a/packages/world-local/src/storage/slot-identity.test.ts b/packages/world-local/src/storage/slot-identity.test.ts new file mode 100644 index 0000000000..00d8b71cce --- /dev/null +++ b/packages/world-local/src/storage/slot-identity.test.ts @@ -0,0 +1,281 @@ +import { promises as fs } from 'node:fs'; +import os from 'node:os'; +import path from 'node:path'; +import { SlotConflictError } from '@workflow/errors'; +import type { Storage } from '@workflow/world'; +import { + FIRST_SLOT, + maxSlotOf, + SPEC_VERSION_CURRENT, + SPEC_VERSION_SLOT_IDENTITY, + slotEventId, + slotFromId, +} from '@workflow/world'; +import { afterEach, beforeEach, describe, expect, it } from 'vitest'; +import { createStorage } from './index.js'; + +let testDir: string; +let storage: Storage; + +beforeEach(async () => { + testDir = await fs.mkdtemp(path.join(os.tmpdir(), 'slot-identity-')); + storage = createStorage(testDir); +}); + +afterEach(async () => { + await fs.rm(testDir, { recursive: true, force: true }); +}); + +/** Start a run whose events are numbered by slot, and return its id. */ +async function newSlotRun(): Promise { + const result = await storage.events.create(null, { + eventType: 'run_created', + specVersion: SPEC_VERSION_SLOT_IDENTITY, + eventData: { + deploymentId: 'dpl_test', + workflowName: 'test-workflow', + input: new Uint8Array(), + }, + }); + if (!result.run) { + throw new Error('Expected run to be created'); + } + return result.run.runId; +} + +/** + * The slots of the run's log, in list order. The page size is explicit: the + * default would silently truncate a fan-out and make a dense log look sparse. + */ +async function slotsOf(runId: string): Promise { + const { data } = await eventsOf(runId); + return data.map((event) => slotFromId(event.eventId) ?? -1); +} + +function eventsOf(runId: string) { + return storage.events.list({ runId, pagination: { limit: 500 } }); +} + +async function createStep( + runId: string, + stepId: string, + eventId?: string +): Promise { + const result = await storage.events.create( + runId, + { + eventType: 'step_created', + specVersion: SPEC_VERSION_SLOT_IDENTITY, + correlationId: stepId, + eventData: { stepName: 'a-step', input: new Uint8Array() }, + }, + eventId === undefined ? undefined : { eventId } + ); + if (!result.event) { + throw new Error('Expected an event'); + } + return result.event.eventId; +} + +describe('numbering', () => { + it('puts run_created in the first slot', async () => { + const runId = await newSlotRun(); + await expect(slotsOf(runId)).resolves.toEqual([FIRST_SLOT]); + }); + + it('allocates dense slots for writers that hold no log', async () => { + // A step completion reporting in, a cancellation from an API call: the + // caller has no event log, so the world numbers the event for it. + const runId = await newSlotRun(); + await createStep(runId, 'step_a'); + await createStep(runId, 'step_b'); + await expect(slotsOf(runId)).resolves.toEqual([1, 2, 3]); + }); + + it('honours a slot the caller claims', async () => { + const runId = await newSlotRun(); + const eventId = await createStep(runId, 'step_a', slotEventId(2)); + expect(eventId).toBe(slotEventId(2)); + await expect(slotsOf(runId)).resolves.toEqual([1, 2]); + }); + + it('keeps a burst of concurrent writers dense', async () => { + // The suspension flush issues every op at once. Density is what lets a + // reader prove its log is complete, so a burst must not leave holes. + const runId = await newSlotRun(); + const ids = await Promise.all( + Array.from({ length: 20 }, (_, index) => + createStep(runId, `step_${index}`) + ) + ); + expect(new Set(ids).size).toBe(ids.length); + const slots = await slotsOf(runId); + expect([...slots].sort((a, b) => a - b)).toEqual( + Array.from({ length: ids.length + 1 }, (_, index) => FIRST_SLOT + index) + ); + }); + + it('proves completeness: the highest slot is the event count', async () => { + const runId = await newSlotRun(); + await Promise.all( + Array.from({ length: 5 }, (_, index) => + createStep(runId, `step_${index}`) + ) + ); + const { data } = await eventsOf(runId); + expect(maxSlotOf(data)).toBe(data.length); + }); + + it('leaves no hole behind a rejected write', async () => { + // The rejected op's slot sits below its concurrent sibling's, and a hole + // below a published event can never be filled. + const runId = await newSlotRun(); + const [rejected, accepted] = await Promise.allSettled([ + storage.events.create(runId, { + eventType: 'step_completed', + specVersion: SPEC_VERSION_SLOT_IDENTITY, + correlationId: 'step_never_created', + eventData: { output: new Uint8Array() }, + }), + createStep(runId, 'step_a'), + ]); + expect(rejected.status).toBe('rejected'); + expect(accepted.status).toBe('fulfilled'); + await createStep(runId, 'step_b'); + const slots = await slotsOf(runId); + expect([...slots].sort((a, b) => a - b)).toEqual([1, 2, 3]); + }); +}); + +describe('mode is pinned to the run', () => { + it('rejects a slot id claimed on a ULID-numbered run', async () => { + const created = await storage.events.create(null, { + eventType: 'run_created', + specVersion: SPEC_VERSION_CURRENT, + eventData: { + deploymentId: 'dpl_test', + workflowName: 'test-workflow', + input: new Uint8Array(), + }, + }); + const runId = created.run?.runId as string; + await expect(createStep(runId, 'step_a', slotEventId(2))).rejects.toThrow( + /not numbered by slot/ + ); + }); + + it('rejects a ULID id claimed on a slot-numbered run', async () => { + const runId = await newSlotRun(); + await expect( + createStep(runId, 'step_a', 'evnt_01K5Z0000000000000000000AA') + ).rejects.toThrow(/not a slot id/); + }); + + it('ignores the spec version of later requests', async () => { + // A run is in exactly one mode for life; only what was persisted decides. + const runId = await newSlotRun(); + const result = await storage.events.create(runId, { + eventType: 'step_created', + specVersion: SPEC_VERSION_CURRENT, + correlationId: 'step_a', + eventData: { stepName: 'a-step', input: new Uint8Array() }, + }); + expect(slotFromId(result.event?.eventId ?? '')).toBe(2); + }); +}); + +describe('conflict', () => { + it('reports the events the loser is missing', async () => { + const runId = await newSlotRun(); + // Out of band: something else takes the slot this caller was about to + // claim, so the caller's log is provably missing an event. + await createStep(runId, 'step_out_of_band'); + + const conflict = await storage.events + .create( + runId, + { + eventType: 'step_created', + specVersion: SPEC_VERSION_SLOT_IDENTITY, + correlationId: 'step_a', + eventData: { stepName: 'a-step', input: new Uint8Array() }, + }, + { eventId: slotEventId(2), maxSlot: 1 } + ) + .catch((error: unknown) => error); + + expect(SlotConflictError.is(conflict)).toBe(true); + const slotConflict = conflict as SlotConflictError; + expect(slotConflict.status).toBe(409); + expect(slotConflict.eventId).toBe(slotEventId(2)); + expect(slotConflict.events?.map((event) => event.eventId)).toEqual([ + slotEventId(2), + ]); + }); + + it('lets the loser re-propose at the next free slot', async () => { + const runId = await newSlotRun(); + await createStep(runId, 'step_out_of_band'); + await expect(createStep(runId, 'step_a', slotEventId(2))).rejects.toThrow( + SlotConflictError + ); + // Merging the delta moves the caller's own numbering forward by one. + const eventId = await createStep(runId, 'step_a', slotEventId(3)); + expect(eventId).toBe(slotEventId(3)); + await expect(slotsOf(runId)).resolves.toEqual([1, 2, 3]); + }); + + it('excludes events the loser already holds from the delta', async () => { + const runId = await newSlotRun(); + await createStep(runId, 'step_one'); + await createStep(runId, 'step_two'); + + const conflict = await storage.events + .create( + runId, + { + eventType: 'step_created', + specVersion: SPEC_VERSION_SLOT_IDENTITY, + correlationId: 'step_a', + eventData: { stepName: 'a-step', input: new Uint8Array() }, + }, + { eventId: slotEventId(2), maxSlot: 2 } + ) + .catch((error: unknown) => error); + + // Slots 1 and 2 are at or below what the caller had; only 3 is news. + expect( + (conflict as SlotConflictError).events?.map((event) => event.eventId) + ).toEqual([slotEventId(3)]); + }); + + it('conflicts across two storage instances sharing a directory', async () => { + // Two instances keep independent books, so the exclusive write — not the + // book — is what decides who owns a slot. + const runId = await newSlotRun(); + const other = createStorage(testDir); + const [first, second] = await Promise.allSettled([ + storage.events.create(runId, { + eventType: 'step_created', + specVersion: SPEC_VERSION_SLOT_IDENTITY, + correlationId: 'step_a', + eventData: { stepName: 'a-step', input: new Uint8Array() }, + }), + other.events.create(runId, { + eventType: 'step_created', + specVersion: SPEC_VERSION_SLOT_IDENTITY, + correlationId: 'step_b', + eventData: { stepName: 'b-step', input: new Uint8Array() }, + }), + ]); + const outcomes = [first, second]; + expect(outcomes.filter((o) => o.status === 'fulfilled')).toHaveLength(1); + const rejection = outcomes.find((o) => o.status === 'rejected'); + expect( + SlotConflictError.is( + (rejection as PromiseRejectedResult | undefined)?.reason + ) + ).toBe(true); + await expect(slotsOf(runId)).resolves.toEqual([1, 2]); + }); +}); diff --git a/packages/world-local/src/storage/slots.test.ts b/packages/world-local/src/storage/slots.test.ts new file mode 100644 index 0000000000..42c962f128 --- /dev/null +++ b/packages/world-local/src/storage/slots.test.ts @@ -0,0 +1,241 @@ +import fs from 'node:fs/promises'; +import os from 'node:os'; +import path from 'node:path'; +import { + FIRST_SLOT, + SPEC_VERSION_CURRENT, + SPEC_VERSION_SLOT_IDENTITY, + slotEventId, +} from '@workflow/world'; +import { afterEach, beforeEach, describe, expect, it } from 'vitest'; +import { createSlotBook, RUN_CREATED_SLOT } from './slots.js'; + +let basedir: string; + +beforeEach(async () => { + basedir = await fs.mkdtemp(path.join(os.tmpdir(), 'slot-book-')); +}); + +afterEach(async () => { + await fs.rm(basedir, { recursive: true, force: true }); +}); + +const RUN_ID = 'wrun_01K0000000000000000000TEST'; + +async function writeRun(specVersion: number): Promise { + await fs.mkdir(path.join(basedir, 'runs'), { recursive: true }); + await fs.writeFile( + path.join(basedir, 'runs', `${RUN_ID}.json`), + JSON.stringify({ + runId: RUN_ID, + deploymentId: 'dpl_test', + status: 'running', + workflowName: 'test', + specVersion, + input: [], + attributes: {}, + createdAt: new Date().toISOString(), + updatedAt: new Date().toISOString(), + }) + ); +} + +async function writeEvents(...slots: number[]): Promise { + await fs.mkdir(path.join(basedir, 'events'), { recursive: true }); + for (const slot of slots) { + await fs.writeFile( + path.join(basedir, 'events', `${RUN_ID}-${slotEventId(slot)}.json`), + '{}' + ); + } +} + +describe('usesSlots', () => { + it('reads the mode off the persisted run, not the build', async () => { + await writeRun(SPEC_VERSION_SLOT_IDENTITY); + await expect(createSlotBook(basedir).usesSlots(RUN_ID)).resolves.toBe(true); + + await writeRun(SPEC_VERSION_CURRENT); + await expect(createSlotBook(basedir).usesSlots(RUN_ID)).resolves.toBe( + false + ); + }); + + it('re-reads until the run exists', async () => { + // The resilient-start path writes run_started before the run entity, so a + // cached "no" taken from the missing run would strand a slot-numbered run + // on ULID ids for the rest of the process's life. + const book = createSlotBook(basedir); + await expect(book.usesSlots(RUN_ID)).resolves.toBe(false); + await writeRun(SPEC_VERSION_SLOT_IDENTITY); + await expect(book.usesSlots(RUN_ID)).resolves.toBe(true); + }); + + it('prefers its own tagged run over the untagged one', async () => { + await writeRun(SPEC_VERSION_SLOT_IDENTITY); + await fs.rename( + path.join(basedir, 'runs', `${RUN_ID}.json`), + path.join(basedir, 'runs', `${RUN_ID}.mine.json`) + ); + await writeRun(SPEC_VERSION_CURRENT); + + await expect( + createSlotBook(basedir, 'mine').usesSlots(RUN_ID) + ).resolves.toBe(true); + await expect( + createSlotBook(basedir, 'other').usesSlots(RUN_ID) + ).resolves.toBe(false); + }); +}); + +describe('reserve', () => { + it('starts at the first slot for a run with no events', async () => { + await expect(createSlotBook(basedir).reserve(RUN_ID)).resolves.toBe( + FIRST_SLOT + ); + }); + + it('continues above the highest slot already on disk', async () => { + await writeEvents(1, 2, 3); + await expect(createSlotBook(basedir).reserve(RUN_ID)).resolves.toBe(4); + }); + + it('fills a hole left in the persisted log', async () => { + // Density is the whole point of the scheme, so a gap that somehow exists is + // reclaimed rather than skipped over forever. + await writeEvents(1, 3); + const book = createSlotBook(basedir); + await expect(book.reserve(RUN_ID)).resolves.toBe(2); + await expect(book.reserve(RUN_ID)).resolves.toBe(4); + }); + + it('hands a synchronous burst distinct consecutive slots', async () => { + // The suspension flush issues every op concurrently; with a plain + // "max + 1" they would all pick the same slot and all but one would fail. + const book = createSlotBook(basedir); + const slots = await Promise.all( + Array.from({ length: 20 }, () => book.reserve(RUN_ID)) + ); + expect([...slots].sort((a, b) => a - b)).toEqual( + Array.from({ length: 20 }, (_, index) => FIRST_SLOT + index) + ); + }); + + it('shares one disk scan across concurrent first callers', async () => { + await writeEvents(1); + const book = createSlotBook(basedir); + const slots = await Promise.all([ + book.reserve(RUN_ID), + book.reserve(RUN_ID), + ]); + expect([...slots].sort((a, b) => a - b)).toEqual([2, 3]); + }); + + it('honours a floor, so a concurrent event cannot take run_created’s slot', async () => { + // start() publishes the run entity before its `run_created` event and + // issues the queue send in parallel, so the delivery's `run_started` can + // allocate while slot 1 is still in flight. + const book = createSlotBook(basedir); + await expect(book.reserve(RUN_ID, RUN_CREATED_SLOT + 1)).resolves.toBe( + RUN_CREATED_SLOT + 1 + ); + }); + + it('leaves slots below the floor allocatable', async () => { + // A floored search skips that range without looking at it, so it proves + // nothing about it — `run_created` must still find its own slot free. + const book = createSlotBook(basedir); + await expect(book.reserve(RUN_ID, RUN_CREATED_SLOT + 1)).resolves.toBe( + RUN_CREATED_SLOT + 1 + ); + await expect(book.reserve(RUN_ID, RUN_CREATED_SLOT)).resolves.toBe( + RUN_CREATED_SLOT + ); + }); + + it('keeps runs independent', async () => { + await writeEvents(1, 2); + const book = createSlotBook(basedir); + await expect(book.reserve(RUN_ID)).resolves.toBe(3); + await expect(book.reserve('wrun_01K0000000000000000000OTHR')).resolves.toBe( + FIRST_SLOT + ); + }); +}); + +describe('release', () => { + it('gives an abandoned interior slot to the next caller', async () => { + // A rejected op must not strand the slot below its concurrent siblings': + // that hole can never be filled once a later slot is published. + const book = createSlotBook(basedir); + const [first, second] = await Promise.all([ + book.reserve(RUN_ID), + book.reserve(RUN_ID), + ]); + book.release(RUN_ID, first); + await expect(book.reserve(RUN_ID)).resolves.toBe(first); + await expect(book.reserve(RUN_ID)).resolves.toBe(second + 1); + }); + + it('does not resurrect a slot that was published', async () => { + const book = createSlotBook(basedir); + const slot = await book.reserve(RUN_ID); + book.observe(RUN_ID, slotEventId(slot)); + book.release(RUN_ID, slot); + await expect(book.reserve(RUN_ID)).resolves.toBe(slot + 1); + }); +}); + +describe('isWritten', () => { + it('reads the log to answer for a run it has never seen', async () => { + await writeEvents(1, 2); + const book = createSlotBook(basedir); + await expect(book.isWritten(RUN_ID, 2)).resolves.toBe(true); + await expect(book.isWritten(RUN_ID, 3)).resolves.toBe(false); + }); + + it('is false for a slot that is only reserved', async () => { + // A reservation is not a publish, so a caller claiming the slot has to be + // allowed through to the write that actually decides it. + const book = createSlotBook(basedir); + const slot = await book.reserve(RUN_ID); + await expect(book.isWritten(RUN_ID, slot)).resolves.toBe(false); + }); +}); + +describe('observe', () => { + it('never hands out a slot claimed by the client', async () => { + const book = createSlotBook(basedir); + await book.reserve(RUN_ID); + book.observe(RUN_ID, slotEventId(5)); + const next = await book.reserve(RUN_ID); + expect(next).not.toBe(5); + expect(next).toBe(2); + }); + + it('ignores ULID event ids', async () => { + const book = createSlotBook(basedir); + await book.reserve(RUN_ID); + book.observe(RUN_ID, 'evnt_01K5Z0000000000000000000AA'); + await expect(book.reserve(RUN_ID)).resolves.toBe(2); + }); +}); + +describe('forget', () => { + it("re-reads the log, picking up another writer's events", async () => { + const book = createSlotBook(basedir); + await expect(book.reserve(RUN_ID)).resolves.toBe(FIRST_SLOT); + await writeEvents(1, 2, 3); + book.forget(RUN_ID); + await expect(book.reserve(RUN_ID)).resolves.toBe(4); + }); + + it('clear() forgets every run', async () => { + await writeEvents(1); + const book = createSlotBook(basedir); + await expect(book.reserve(RUN_ID)).resolves.toBe(2); + await writeEvents(2, 3); + book.clear(); + await expect(book.reserve(RUN_ID)).resolves.toBe(4); + }); +}); diff --git a/packages/world-local/src/storage/slots.ts b/packages/world-local/src/storage/slots.ts new file mode 100644 index 0000000000..c6b2f017f6 --- /dev/null +++ b/packages/world-local/src/storage/slots.ts @@ -0,0 +1,247 @@ +/** + * Slot allocation for the Local World. + * + * A slot-numbered run names its events by position: `evnt_…001` is the first + * event of the run, `evnt_…002` the second, with no gaps. Density is the point + * — it is what lets a reader prove its loaded log is complete — so an allocator + * must never leave a slot permanently unwritten. + * + * Three properties do the work: + * + * - Handing out a slot is a *synchronous* set operation, so concurrent + * callers in one process get distinct slots with no lock. The only await is + * seeding from disk, which is memoized per run. + * - An allocation always picks the lowest slot that is neither written nor + * outstanding, so a reservation that is abandoned (its create threw a + * validation error) is handed to the next caller instead of leaving an + * interior hole. This matters under fan-out: N concurrent step_completed + * writes reserve N consecutive slots, and one rejected op must not strand + * the slot below its siblings'. + * - The event publish is `writeExclusive`, which is the authority. The book is + * a hint: when it turns out to be stale (another process wrote the slot), + * the publish fails and the caller is told so, rather than a duplicate being + * written or a slot being skipped. + * + * The book is per storage instance, and two instances may share a data + * directory (the cross-process convergence tests rely on exactly that). Their + * books are then independent, and the loser of a collision gets a conflict it + * has to resolve by reloading — the same contract as the networked worlds. + */ + +import type { WorkflowRun } from '@workflow/world'; +import { + FIRST_SLOT, + slotFromId, + usesSlotIdentity, + WorkflowRunSchema, +} from '@workflow/world'; +import { readJSONWithFallback } from '../fs.js'; +import { listRunEventIds } from './helpers.js'; + +interface RunSlots { + /** Slots proven to be on disk. */ + written: Set; + /** Slots handed out whose publish has not resolved yet. */ + outstanding: Set; + /** Lowest slot that might still be free; never decreases except on release. */ + searchFrom: number; +} + +export interface SlotBook { + /** + * Whether `runId`'s events are numbered by slot, read from the run's + * persisted `specVersion` — never from the build, so a run stays in the mode + * it was created in for life. A run that does not exist yet is not + * slot-numbered, and that answer is not cached: the resilient-start path + * creates the run moments later, and caching "no" would strand it on ULIDs + * for the rest of this process's life. + */ + usesSlots(runId: string): Promise; + /** + * Reserves the lowest free slot of `runId`, at or above `minSlot`. Distinct + * for every concurrent caller; the publish still has to prove the slot was + * actually free. + * + * `minSlot` is how the run's first slot is kept for its own `run_created`: + * that event's slot needs no allocation, and it may not be on disk yet when a + * concurrent `run_started` allocates (start() issues the creation and the + * queue send in parallel, and the run entity is published before its event). + * Slots below `minSlot` stay allocatable for a later caller, so holding one + * back leaves the log dense. + */ + reserve(runId: string, minSlot?: number): Promise; + /** + * Records that a caller claimed `slot` itself, so an allocation running + * alongside it picks a different one. Reserved and released on the same terms + * as {@link reserve}: the claim is only a hint until the publish proves it. + */ + claim(runId: string, slot: number): void; + /** + * Whether `slot` is already occupied by a published event, seeding from disk + * if this run has not been read yet. + * + * Lets a doomed claim be rejected *before* the create materializes its step, + * hook or wait: the entity mutation runs ahead of the event publish, so a + * claim that only fails at the publish leaves an entity behind with no event, + * and the caller's re-proposal at the next slot then collides with its own + * orphan. A `false` here is not a promise — the publish is still the + * authority — but it turns the case that actually happens (a caller numbering + * from a stale log) into a clean conflict. + */ + isWritten(runId: string, slot: number): Promise; + /** + * Returns a reserved or claimed slot that was never published, so the next + * caller takes it instead of it becoming a hole. + */ + release(runId: string, slot: number): void; + /** Records a published event id, so it is never handed out again. */ + observe(runId: string, eventId: string): void; + /** Drops what is cached for `runId`, so the next reservation re-reads disk. */ + forget(runId: string): void; + /** Drops everything cached (the data directory was cleared out from under us). */ + clear(): void; +} + +export function createSlotBook(basedir: string, tag?: string): SlotBook { + /** runId → whether the run is slot-numbered, memoized once it exists. */ + const modes = new Map(); + const books = new Map(); + /** runId → in-flight seed scan, so concurrent first callers share one scan. */ + const seeds = new Map>(); + + async function readMode(runId: string): Promise { + const run = await readJSONWithFallback( + basedir, + 'runs', + runId, + WorkflowRunSchema, + tag + ); + return run ? usesSlotIdentity(run.specVersion) : false; + } + + async function seed(runId: string): Promise { + const eventIds = await listRunEventIds(basedir, runId, tag); + const written = new Set(); + for (const eventId of eventIds) { + const slot = slotFromId(eventId); + if (slot !== undefined) { + written.add(slot); + } + } + const book: RunSlots = { + written, + outstanding: new Set(), + searchFrom: FIRST_SLOT, + }; + books.set(runId, book); + return book; + } + + /** The run's book, seeding it from disk once for all concurrent callers. */ + function open(runId: string): RunSlots | Promise { + const known = books.get(runId); + if (known) { + return known; + } + let pending = seeds.get(runId); + if (!pending) { + pending = seed(runId).finally(() => seeds.delete(runId)); + seeds.set(runId, pending); + } + return pending; + } + + function take(book: RunSlots, minSlot: number): number { + let slot = Math.max(book.searchFrom, minSlot); + while (book.written.has(slot) || book.outstanding.has(slot)) { + slot += 1; + } + book.outstanding.add(slot); + if (minSlot <= book.searchFrom) { + // Only an unfloored search proves everything below the slot it landed on + // is taken. A floored one skipped that range without looking, and those + // slots are still free for a caller that may take them. + book.searchFrom = slot; + } + return slot; + } + + return { + async usesSlots(runId) { + const cached = modes.get(runId); + if (cached !== undefined) { + return cached; + } + const mode = await readMode(runId); + // `false` here can mean "run not created yet" as well as "ULID run", and + // only the run's own absence is transient — so remember the positive + // answer eagerly and re-read until the run exists. + if (mode) { + modes.set(runId, true); + } + return mode; + }, + + async reserve(runId, minSlot = FIRST_SLOT) { + const opened = open(runId); + // Awaiting a book that is already in hand would yield to the microtask + // queue and let a concurrent caller take the same slot. + return take(opened instanceof Promise ? await opened : opened, minSlot); + }, + + claim(runId, slot) { + // No book means nothing is allocating for this run in this instance yet, + // and the seed scan that starts one reads the claim off disk if it landed. + books.get(runId)?.outstanding.add(slot); + }, + + async isWritten(runId, slot) { + const book = await open(runId); + return book.written.has(slot); + }, + + release(runId, slot) { + const book = books.get(runId); + if (!book) { + return; + } + book.outstanding.delete(slot); + if (slot < book.searchFrom) { + book.searchFrom = slot; + } + }, + + observe(runId, eventId) { + const book = books.get(runId); + if (!book) { + // Nothing to keep consistent: the slot is on disk by the time this is + // called, so the eventual seed scan picks it up. + return; + } + const slot = slotFromId(eventId); + if (slot === undefined) { + return; + } + book.written.add(slot); + book.outstanding.delete(slot); + }, + + forget(runId) { + modes.delete(runId); + books.delete(runId); + }, + + clear() { + modes.clear(); + books.clear(); + }, + }; +} + +/** + * The slot a run's first event occupies. A run's own `run_created` is the only + * event that can be numbered without consulting the log, because there is + * provably nothing before it. + */ +export const RUN_CREATED_SLOT = FIRST_SLOT; diff --git a/packages/world-postgres/src/drizzle/migrations/0018_run_scoped_event_and_step_keys.sql b/packages/world-postgres/src/drizzle/migrations/0018_run_scoped_event_and_step_keys.sql new file mode 100644 index 0000000000..0764dee02d --- /dev/null +++ b/packages/world-postgres/src/drizzle/migrations/0018_run_scoped_event_and_step_keys.sql @@ -0,0 +1,13 @@ +-- Event ids and step ids are unique per run, not globally. Under slot identity +-- (spec 6) every run numbers its own log from 1, so "evnt_0...001" and +-- "step_0...001" exist once per run and the old global primary keys would make +-- the second run to reach slot 1 collide with the first. +-- +-- The run leads both keys so the existing run-scoped range scans stay a single +-- index seek; that also makes the standalone run_id indexes redundant. +ALTER TABLE "workflow"."workflow_events" DROP CONSTRAINT IF EXISTS "workflow_events_pkey";--> statement-breakpoint +ALTER TABLE "workflow"."workflow_events" ADD CONSTRAINT "workflow_events_run_id_id_pk" PRIMARY KEY("run_id","id");--> statement-breakpoint +DROP INDEX IF EXISTS "workflow"."workflow_events_run_id_index";--> statement-breakpoint +ALTER TABLE "workflow"."workflow_steps" DROP CONSTRAINT IF EXISTS "workflow_steps_pkey";--> statement-breakpoint +ALTER TABLE "workflow"."workflow_steps" ADD CONSTRAINT "workflow_steps_run_id_step_id_pk" PRIMARY KEY("run_id","step_id");--> statement-breakpoint +DROP INDEX IF EXISTS "workflow"."workflow_steps_run_id_index"; diff --git a/packages/world-postgres/src/drizzle/migrations/meta/_journal.json b/packages/world-postgres/src/drizzle/migrations/meta/_journal.json index b7fb5d8215..9dca967a5b 100644 --- a/packages/world-postgres/src/drizzle/migrations/meta/_journal.json +++ b/packages/world-postgres/src/drizzle/migrations/meta/_journal.json @@ -127,6 +127,13 @@ "when": 1785283200000, "tag": "0017_add_hook_resume_context", "breakpoints": true + }, + { + "idx": 18, + "version": "7", + "when": 1785801600000, + "tag": "0018_run_scoped_event_and_step_keys", + "breakpoints": true } ] } diff --git a/packages/world-postgres/src/drizzle/schema.ts b/packages/world-postgres/src/drizzle/schema.ts index 6ffb21abcb..df58f73eb8 100644 --- a/packages/world-postgres/src/drizzle/schema.ts +++ b/packages/world-postgres/src/drizzle/schema.ts @@ -133,7 +133,7 @@ export const runs = schema.table( export const events = schema.table( 'workflow_events', { - eventId: varchar('id').primaryKey(), + eventId: varchar('id').notNull(), eventType: varchar('type').$type().notNull(), correlationId: varchar('correlation_id'), createdAt: timestamp('created_at').defaultNow().notNull(), @@ -146,7 +146,11 @@ export const events = schema.table( Cborized & { eventData?: undefined }, 'eventData'> >, (tb) => [ - index().on(tb.runId), + // Event ids are only unique within their run: under slot identity every run + // numbers its own log from 1, so `evnt_0…001` exists once per run. The run + // leads the key so the range scans in `list` stay a single index seek, and + // it subsumes the plain `run_id` index the table used to carry. + primaryKey({ columns: [tb.runId, tb.eventId] }), index().on(tb.correlationId), // Runtime-correlated one-shot events must be unique per (run, correlation) // — without @@ -167,7 +171,7 @@ export const steps = schema.table( 'workflow_steps', { runId: varchar('run_id').notNull(), - stepId: varchar('step_id').primaryKey(), + stepId: varchar('step_id').notNull(), stepName: varchar('step_name').notNull(), status: stepStatus('status').notNull(), /** @deprecated */ @@ -203,7 +207,13 @@ export const steps = schema.table( 'output' | 'input' | 'error' > >, - (tb) => [index().on(tb.runId), index().on(tb.status)] + (tb) => [ + // A step id is a correlation id, which under slot identity is only unique + // within its run — same reasoning as `workflow_events`. Every step query in + // this world is already run-scoped, so the run leads the key. + primaryKey({ columns: [tb.runId, tb.stepId] }), + index().on(tb.status), + ] ); export const hooks = schema.table( diff --git a/packages/world-postgres/src/index.ts b/packages/world-postgres/src/index.ts index 430cea0812..0db47b89e2 100644 --- a/packages/world-postgres/src/index.ts +++ b/packages/world-postgres/src/index.ts @@ -1,5 +1,5 @@ import type { Storage, World } from '@workflow/world'; -import { reenqueueActiveRuns, SPEC_VERSION_CURRENT } from '@workflow/world'; +import { mintedSpecVersion, reenqueueActiveRuns } from '@workflow/world'; import { Pool } from 'pg'; import type { PostgresWorldConfig } from './config.js'; import { createClient, type Drizzle } from './drizzle/index.js'; @@ -63,7 +63,10 @@ export function createWorld( const streamer = createStreamer(pool, drizzle); return { - specVersion: SPEC_VERSION_CURRENT, + // What this world stamps on new runs, which is not the newest version it + // can read: slot identity is readable everywhere and minted only where + // WORKFLOW_SLOT_IDENTITY is set. + specVersion: mintedSpecVersion(), ...storage, ...streamer, ...queue, diff --git a/packages/world-postgres/src/slots.ts b/packages/world-postgres/src/slots.ts new file mode 100644 index 0000000000..89435810a9 --- /dev/null +++ b/packages/world-postgres/src/slots.ts @@ -0,0 +1,225 @@ +/** + * Slot identity for the postgres world. + * + * A slot-numbered run names its events by position: `evnt_…001` is the first + * event of the run, `evnt_…002` the second, with no gaps. Density is the point + * — it is what lets a reader prove its copy of a log is complete — so the two + * things this module has to get right are that a position is written at most + * once and that a lost position is never left behind as a hole. + * + * The authority for both is the events table's primary key, `(run_id, id)`: the + * INSERT either lands or raises a unique violation, and a writer that loses the + * race is retried at a position that is still free rather than abandoning the + * one it lost. The probe below is only ever a hint about where to try next. + */ + +import { WorkflowWorldError } from '@workflow/errors'; +import { FIRST_SLOT, slotEventId, slotFromId } from '@workflow/world'; +import { and, desc, eq } from 'drizzle-orm'; +import { type Drizzle, Schema } from './drizzle/index.js'; + +/** + * The slot a run's own `run_created` occupies. Nothing in a run precedes its + * creation, so this one position needs no allocation, and every other event of + * the run searches above it — including the event that happens to reach storage + * first, which on the start path is routinely `run_started`. + */ +export const RUN_CREATED_SLOT = FIRST_SLOT; + +/** First backoff after losing a position; doubled each round. */ +export const SLOT_RETRY_BASE_MS = 5; + +/** Ceiling for a single backoff, so a contended run keeps making attempts. */ +export const SLOT_RETRY_MAX_DELAY_MS = 250; + +/** + * How long a writer keeps looking for a free position before giving up. + * Exhausting it surfaces as a 503, so the caller — in practice a queue + * delivery — retries the whole operation instead of the run stalling on it. + */ +export const SLOT_RETRY_BUDGET_MS = 30_000; + +/** Postgres unique-violation code. */ +const UNIQUE_VIOLATION = '23505'; + +/** + * Whether an error says the position a write aimed at is already occupied. + * + * Drizzle wraps the pg error, so the code can sit on the error or on its cause. + * Both the name drizzle generates for the composite key and the name postgres + * gives an inline `PRIMARY KEY` are accepted, so a database whose key predates + * the run-scoped migration still classifies correctly. + */ +export function isEventKeyViolation(error: unknown): boolean { + const pg = (error as { code?: string; constraint?: string }).code + ? (error as { code?: string; constraint?: string }) + : ((error as { cause?: { code?: string; constraint?: string } }).cause ?? + {}); + return ( + pg.code === UNIQUE_VIOLATION && + (pg.constraint === 'workflow_events_run_id_id_pk' || + pg.constraint === 'workflow_events_pkey') + ); +} + +/** + * The highest event id in a run's log, or undefined when the log is empty. + * + * One backwards scan of the `(run_id, id)` primary key. Ids are fixed-width + * within a scheme, so for a slot-numbered run the highest id names the highest + * written position — and because a log holds ids of exactly one scheme, that id + * also reports which scheme the run was created with. + */ +export async function highestEventId( + drizzle: Drizzle, + runId: string +): Promise { + const [row] = await drizzle + .select({ eventId: Schema.events.eventId }) + .from(Schema.events) + .where(eq(Schema.events.runId, runId)) + .orderBy(desc(Schema.events.eventId)) + .limit(1); + return row?.eventId; +} + +/** The position an id names, or 0 for an empty log or a ULID-numbered one. */ +export function highestSlotOf(eventId: string | undefined): number { + return eventId === undefined ? 0 : (slotFromId(eventId) ?? 0); +} + +/** Whether a run's log already holds `eventId`. */ +export async function eventExists( + drizzle: Drizzle, + runId: string, + eventId: string +): Promise { + const [row] = await drizzle + .select({ eventId: Schema.events.eventId }) + .from(Schema.events) + .where( + and(eq(Schema.events.runId, runId), eq(Schema.events.eventId, eventId)) + ) + .limit(1); + return row !== undefined; +} + +/** Full jitter over an exponentially growing, capped window. */ +export function slotRetryDelay(round: number): number { + return ( + Math.random() * + Math.min(SLOT_RETRY_BASE_MS * 2 ** round, SLOT_RETRY_MAX_DELAY_MS) + ); +} + +/** The event ids a single create publishes. */ +export interface EventIds { + /** + * The id of the event this create returns. Taken on demand: a ULID-numbered + * write mints it inside its transaction, once the row lock that orders it is + * held. + */ + primary: () => string; + /** + * An additional event written in the same breath — the synthetic + * `step_created` of a lazy step start. Its position is allocated here even + * when the caller named its own for the primary event, because a caller that + * defers a `step_created` cannot know it will be synthesized. + */ + extra: () => Promise; +} + +export interface PlaceEventOptions { + /** + * Position the caller named, when it holds the log and claimed one. A claim + * asserts the log is complete up to that position, so losing it is a conflict + * the caller has to resolve rather than something to retry here. + */ + claimedSlot?: number; + /** Lowest position this write may take when allocating. */ + minSlot: number; + /** + * Result of a probe the caller has already made, used for the first attempt + * instead of probing again. Later rounds always re-probe: the log has + * demonstrably moved. + */ + seedHighestEventId?: string | undefined; + /** The conflict raised when a claimed position turns out to be taken. */ + onClaimTaken: () => Promise; + /** Performs the write with the ids it should publish under. */ + write: (ids: EventIds) => Promise; +} + +/** + * Writes an event of a slot-numbered run, at the position the caller claimed or + * at the next free one. + * + * Every round re-probes rather than incrementing a local counter: each round at + * least one writer wins, so re-probing guarantees progress under any amount of + * contention. `write` must leave nothing behind when it raises a unique + * violation — the callers here either write only the event row or wrap their + * materialization in the same transaction, so a lost round rolls back whole. + */ +export async function placeEvent( + drizzle: Drizzle, + runId: string, + options: PlaceEventOptions +): Promise { + const deadline = Date.now() + SLOT_RETRY_BUDGET_MS; + for (let round = 0; ; round++) { + let cursor: number | undefined; + /** + * Positions for this attempt, consecutive from one probe. Deferred so a + * claimed write with no extra event never probes at all. + */ + const take = async (): Promise => { + if (cursor === undefined) { + const highest = + round === 0 && options.seedHighestEventId !== undefined + ? options.seedHighestEventId + : await highestEventId(drizzle, runId); + cursor = Math.max( + highestSlotOf(highest) + 1, + options.minSlot, + // The claimed position is this write's own; an extra event must not + // be handed it. + (options.claimedSlot ?? 0) + 1 + ); + } + return cursor++; + }; + + const primary = + options.claimedSlot === undefined + ? slotEventId(await take()) + : slotEventId(options.claimedSlot); + try { + return await options.write({ + primary: () => primary, + extra: async () => slotEventId(await take()), + }); + } catch (error) { + if (!isEventKeyViolation(error)) { + throw error; + } + // A claimed write can also lose on its extra event's position, which is + // this world's to reallocate — only a claim that is itself taken is the + // caller's problem. + if ( + options.claimedSlot !== undefined && + (await eventExists(drizzle, runId, primary)) + ) { + throw await options.onClaimTaken(); + } + if (Date.now() >= deadline) { + throw new WorkflowWorldError( + `Could not place an event in run "${runId}" within ${SLOT_RETRY_BUDGET_MS}ms of contention`, + { status: 503 } + ); + } + await new Promise((resolve) => + setTimeout(resolve, slotRetryDelay(round)) + ); + } + } +} diff --git a/packages/world-postgres/src/storage.ts b/packages/world-postgres/src/storage.ts index 140597a6a9..878f39758b 100644 --- a/packages/world-postgres/src/storage.ts +++ b/packages/world-postgres/src/storage.ts @@ -3,12 +3,14 @@ import { HookNotFoundError, RunExpiredError, RunNotSupportedError, + SlotConflictError, TooEarlyError, WorkflowRunNotFoundError, WorkflowWorldError, } from '@workflow/errors'; import type { AttributeChange, + CreateEventParams, Event, EventResult, ExperimentalSetAttributesResult, @@ -35,15 +37,20 @@ import { isChildEntityCreationEventType, isHookEventRequiringExistence, isLegacySpecVersion, + isSlotId, isTerminalRunEventType, isTerminalStepStatus, isTerminalWorkflowRunStatus, requiresNewerWorld, SPEC_VERSION_CURRENT, + SPEC_VERSION_MAX_SUPPORTED, StepSchema, + slotEventId, + slotFromId, stripEventDataRefs, TERMINAL_STEP_STATUSES, TERMINAL_WORKFLOW_RUN_STATUSES, + usesSlotIdentity, validateAttributeChanges, validateUlidTimestamp, WorkflowRunSchema, @@ -62,6 +69,13 @@ import { import { monotonicFactory } from 'ulid'; import { type Drizzle, Schema } from './drizzle/index.js'; import type { SerializedContent } from './drizzle/schema.js'; +import { + type EventIds, + eventExists, + highestEventId, + placeEvent, + RUN_CREATED_SLOT, +} from './slots.js'; import { compact } from './util.js'; /** @@ -464,6 +478,64 @@ export function createEventsStorage(drizzle: Drizzle): Storage['events'] { .limit(1) .prepare('events_get_wait_for_validation'); + /** + * The events a caller that just lost a slot is missing: one ascending page of + * the run's log after the cursor it wrote from, minus anything at or below the + * highest slot it already held. Because slots are dense, that second filter is + * exact — a caller cannot be missing an event whose position it can name. + * + * Returned inline with the conflict so the common case (a handful of events + * arrived out of band) costs the caller no extra round-trip. `hasMore` is + * forwarded verbatim: an overflowing delta is the caller's signal to page from + * `cursor` instead of treating this as the whole story. + */ + async function eventsAfterClaim( + runId: string, + params: CreateEventParams | undefined + ): Promise<{ events: Event[]; cursor: string | null; hasMore: boolean }> { + const limit = 100; + const all = await drizzle + .select() + .from(events) + .where( + and( + eq(events.runId, runId), + map(params?.sinceCursor, (c) => gt(events.eventId, c)) + ) + ) + .orderBy(events.eventId) + .limit(limit + 1); + const page = all.slice(0, limit); + const maxSlot = params?.maxSlot ?? 0; + const resolveData = params?.resolveData ?? 'all'; + return { + events: page + .filter((v) => (slotFromId(v.eventId) ?? 0) > maxSlot) + .map((v) => { + v.eventData ||= v.eventDataJson; + return stripEventDataRefs(EventSchema.parse(compact(v)), resolveData); + }), + cursor: page.at(-1)?.eventId ?? null, + hasMore: all.length > limit, + }; + } + + /** + * The 409 a caller gets when the slot it named turns out to belong to someone + * else, carrying the events it is missing so it can merge, replay and + * re-propose at a free position. + */ + async function slotConflict( + runId: string, + eventId: string, + params: CreateEventParams | undefined + ): Promise { + return new SlotConflictError( + `Slot ${slotFromId(eventId)} of run "${runId}" is already taken`, + { eventId, ...(await eventsAfterClaim(runId, params)) } + ); + } + return { async create(runId, data, params): Promise { let eventId: string | undefined; @@ -490,6 +562,20 @@ export function createEventsStorage(drizzle: Drizzle): Storage['events'] { // specVersion is always sent by the runtime, but we provide a fallback for safety const effectiveSpecVersion = data.specVersion ?? SPEC_VERSION_CURRENT; + // Whether this run numbers its events by slot. Decided from what was + // persisted, never from this request or this build, so a run stays in the + // mode it was created in for life — a run whose log holds ULID ids must + // never be handed a slot id, and vice versa. `run_created` is the one + // event that decides the mode instead of reading it; the resilient-start + // path below decides it too, on the request that creates the run. + let slotMode: boolean | undefined = + data.eventType === 'run_created' + ? usesSlotIdentity(effectiveSpecVersion) + : undefined; + // The run's highest event id, when it was read before the write. Seeds the + // allocator's first attempt so a probe is never made twice. + let seedHighestEventId: string | undefined; + // Track entity created/updated for EventResult let run: WorkflowRun | undefined; let step: Step | undefined; @@ -585,7 +671,13 @@ export function createEventsStorage(drizzle: Drizzle): Storage['events'] { .returning(); if (inserted) { - const runCreatedEventId = `wevt_${ulid()}`; + // We created the run, so this request also decided its mode. + slotMode = usesSlotIdentity(effectiveSpecVersion); + // A run's own `run_created` provably has nothing before it, so its + // slot needs no allocation. + const runCreatedEventId = slotMode + ? slotEventId(RUN_CREATED_SLOT) + : `wevt_${ulid()}`; await drizzle.insert(events).values({ runId: effectiveRunId, eventId: runCreatedEventId, @@ -631,7 +723,7 @@ export function createEventsStorage(drizzle: Drizzle): Storage['events'] { if (requiresNewerWorld(currentRun.specVersion)) { throw new RunNotSupportedError( currentRun.specVersion!, - SPEC_VERSION_CURRENT + SPEC_VERSION_MAX_SUPPORTED ); } @@ -651,6 +743,92 @@ export function createEventsStorage(drizzle: Drizzle): Storage['events'] { throw new WorkflowRunNotFoundError(effectiveRunId); } + // The run entity is the authority on the mode, and it can appear between + // the resilient-start insert above and this point: start() issues + // `run_created` and the queue send concurrently, so a delivery's + // `run_started` can arrive while the run is still being published. A stale + // "no" would number that one event with a ULID on an otherwise + // slot-numbered run, and the hole it leaves in the numbering costs the log + // its completeness proof for life. + if (currentRun && data.eventType !== 'run_created') { + slotMode = usesSlotIdentity(currentRun.specVersion); + } + if (slotMode === undefined) { + // step_completed and step_retrying skip the run read above. The log's + // own highest id reports the scheme, since a log holds ids of exactly + // one, and it is the probe the allocator needs anyway — so a + // slot-numbered run pays nothing extra for this query. + seedHighestEventId = await highestEventId(drizzle, effectiveRunId); + slotMode = isSlotId(seedHighestEventId ?? ''); + } + + // ============================================================ + // EVENT ID: the caller's slot claim, an allocated slot, or a ULID + // ============================================================ + // A slot-numbered run's ids name positions in its log, so an id is either + // claimed by a caller that holds the log (and is therefore asserting the + // log is complete up to that position) or allocated at write time for a + // caller that has no log — a step completion reporting in, a cancellation + // from an API call. + let claimedSlot: number | undefined; + if (params?.eventId !== undefined) { + if (!slotMode) { + throw new WorkflowWorldError( + `Event id "${params.eventId}" was supplied for run "${effectiveRunId}", whose events are not numbered by slot`, + { status: 400 } + ); + } + claimedSlot = slotFromId(params.eventId); + if (claimedSlot === undefined) { + throw new WorkflowWorldError( + `Event id "${params.eventId}" is not a slot id, and run "${effectiveRunId}" numbers its events by slot`, + { status: 400 } + ); + } + eventId = params.eventId; + if (await eventExists(drizzle, effectiveRunId, eventId)) { + // Reject a doomed claim before the materialization below creates the + // step, hook or wait this event will now never accompany. A caller + // that re-proposes at the next slot would otherwise collide with its + // own orphan and read that as "my write already landed". + throw await slotConflict(effectiveRunId, eventId, params); + } + } + + /** + * Runs one of the event writes below under this run's id discipline: in + * slot mode it places the event at the position the caller claimed or at + * the next free one, retrying a position lost to a concurrent writer; + * otherwise it mints a ULID. + */ + const publish = async ( + write: (ids: EventIds) => Promise + ): Promise => + slotMode + ? placeEvent(drizzle, effectiveRunId, { + ...(claimedSlot !== undefined ? { claimedSlot } : {}), + // A run's own `run_created` owns the first slot — provably, since + // nothing precedes it — so every other event allocates above it, + // even when that event is the first to arrive here. + minSlot: + data.eventType === 'run_created' + ? RUN_CREATED_SLOT + : RUN_CREATED_SLOT + 1, + seedHighestEventId, + onClaimTaken: () => + slotConflict(effectiveRunId, eventId as string, params), + write: (ids) => { + // Each attempt publishes under its own id, and the result and + // error messages below read it back from here. + eventId = ids.primary(); + return write(ids); + }, + }) + : write({ + primary: getEventId, + extra: async () => `wevt_${ulid()}`, + }); + // Lazy step start: a step_started carrying step-creation data // (stepName + input) may arrive with no prior step_created — it creates // the step on the fly (see the materialization block below). This @@ -676,17 +854,19 @@ export function createEventsStorage(drizzle: Drizzle): Storage['events'] { .limit(1); // Create the event (still record it) - const [value] = await drizzle - .insert(Schema.events) - .values({ - runId: effectiveRunId, - eventId: getEventId(), - correlationId: data.correlationId, - eventType: data.eventType, - eventData: 'eventData' in data ? data.eventData : undefined, - specVersion: effectiveSpecVersion, - }) - .returning({ createdAt: Schema.events.createdAt }); + const [value] = await publish((ids) => + drizzle + .insert(Schema.events) + .values({ + runId: effectiveRunId, + eventId: ids.primary(), + correlationId: data.correlationId, + eventType: data.eventType, + eventData: 'eventData' in data ? data.eventData : undefined, + specVersion: effectiveSpecVersion, + }) + .returning({ createdAt: Schema.events.createdAt }) + ); const result = { ...data, @@ -1192,41 +1372,52 @@ export function createEventsStorage(drizzle: Drizzle): Storage['events'] { // event INSERT behind that lock prevents a late step_started from being // ordered after a concurrent terminal event that already won the row. if (data.eventType === 'step_started') { - value = await drizzle.transaction(async (tx) => { - // Lazy step start: no prior step_created exists, but this - // step_started carries the step-creation data. The step INSERT is - // the ownership claim: only the caller that inserts the row gets to - // run the step body inline. - if (lazyStepStart && !validatedStep) { - const lazyData = data.eventData; - const [inserted] = await tx - .insert(Schema.steps) - .values({ - runId: effectiveRunId, - stepId: data.correlationId, - stepName: lazyData.stepName, - input: lazyData.input as SerializedContent, - status: 'pending', - attempt: 0, - specVersion: effectiveSpecVersion, - }) - .onConflictDoNothing() - .returning({ stepId: Schema.steps.stepId }); - - if (!inserted) { - throw new EntityConflictError( - `Step "${data.correlationId}" already created` - ); - } + // The whole transaction is the retry unit here: a step_started that + // loses its slot has to roll the step row and the synthetic + // step_created back with it, or the next attempt would trip its own + // orphaned step and read that as "a concurrent handler won the create". + value = await publish((ids) => + drizzle.transaction(async (tx) => { + // Lazy step start: no prior step_created exists, but this + // step_started carries the step-creation data. The step INSERT is + // the ownership claim: only the caller that inserts the row gets to + // run the step body inline. + if (lazyStepStart && !validatedStep) { + const lazyData = data.eventData; + const [inserted] = await tx + .insert(Schema.steps) + .values({ + runId: effectiveRunId, + stepId: data.correlationId, + stepName: lazyData.stepName, + input: lazyData.input as SerializedContent, + status: 'pending', + attempt: 0, + specVersion: effectiveSpecVersion, + }) + .onConflictDoNothing() + .returning({ stepId: Schema.steps.stepId }); + + if (!inserted) { + throw new EntityConflictError( + `Step "${data.correlationId}" already created` + ); + } - // Replay still needs to observe step_created before - // step_started. Because this synthetic event is in the same - // transaction as the lazy step row and step_started event, we - // cannot leave behind only one side of that materialization. - const stepCreatedEventId = `wevt_${ulid()}`; - await tx - .insert(events) - .values({ + // Replay still needs to observe a step_created at all: the + // client's step consumer sets hasCreatedEvent only on that event + // type. Which of the pair sorts first does not matter — the + // step_started consumer is a no-op — but leaving behind only one + // side of the materialization would, hence the shared + // transaction. + // + // It takes a position of its own, which on a claimed write is + // necessarily one this world allocated: a caller that defers a + // step_created cannot know it will be synthesized, so it never + // claims a slot for it. Losing that position rolls the transaction + // back for a retry, so the insert must not swallow the collision. + const stepCreatedEventId = await ids.extra(); + const insertStepCreated = tx.insert(events).values({ runId: effectiveRunId, eventId: stepCreatedEventId, correlationId: data.correlationId, @@ -1236,99 +1427,104 @@ export function createEventsStorage(drizzle: Drizzle): Storage['events'] { input: lazyData.input, }, specVersion: effectiveSpecVersion, - }) - .onConflictDoNothing(); - stepCreatedLazily = true; - } - - // Retried steps may be scheduled for later. Keep this check inside - // the transaction so the step_started write cannot slip past it. - if ( - validatedStep?.retryAfter && - validatedStep.retryAfter.getTime() > Date.now() - ) { - throw new TooEarlyError( - `Cannot start step "${data.correlationId}": retryAfter timestamp has not been reached yet`, - { - retryAfter: Math.ceil( - (validatedStep.retryAfter.getTime() - Date.now()) / 1000 - ), - } - ); - } + }); + await (slotMode + ? insertStepCreated + : insertStepCreated.onConflictDoNothing()); + stepCreatedLazily = true; + } - // The terminal-state guard is part of the UPDATE, not just the - // earlier validation read. That closes the race where another - // writer completes/fails the step between validation and start. - const [stepValue] = await tx - .update(Schema.steps) - .set({ - status: 'running', - attempt: sql`${Schema.steps.attempt} + 1`, - // Preserve the original first-start timestamp across retries or - // overlapping starts. - startedAt: sql`COALESCE(${Schema.steps.startedAt}, ${now.toISOString()})`, - retryAfter: null, - }) - .where( - and( - eq(Schema.steps.runId, effectiveRunId), - eq(Schema.steps.stepId, data.correlationId!), - notInArray(Schema.steps.status, terminalStepStatuses) - ) - ) - .returning(); + // Retried steps may be scheduled for later. Keep this check inside + // the transaction so the step_started write cannot slip past it. + if ( + validatedStep?.retryAfter && + validatedStep.retryAfter.getTime() > Date.now() + ) { + throw new TooEarlyError( + `Cannot start step "${data.correlationId}": retryAfter timestamp has not been reached yet`, + { + retryAfter: Math.ceil( + (validatedStep.retryAfter.getTime() - Date.now()) / 1000 + ), + } + ); + } - if (stepValue) { - step = deserializeStepError(compact(stepValue)); - } else { - const [existing] = await tx - .select({ status: Schema.steps.status }) - .from(Schema.steps) + // The terminal-state guard is part of the UPDATE, not just the + // earlier validation read. That closes the race where another + // writer completes/fails the step between validation and start. + const [stepValue] = await tx + .update(Schema.steps) + .set({ + status: 'running', + attempt: sql`${Schema.steps.attempt} + 1`, + // Preserve the original first-start timestamp across retries or + // overlapping starts. + startedAt: sql`COALESCE(${Schema.steps.startedAt}, ${now.toISOString()})`, + retryAfter: null, + }) .where( and( eq(Schema.steps.runId, effectiveRunId), - eq(Schema.steps.stepId, data.correlationId!) + eq(Schema.steps.stepId, data.correlationId!), + notInArray(Schema.steps.status, terminalStepStatuses) ) ) - .limit(1); - if (!existing) { - throw new WorkflowWorldError( - `Step "${data.correlationId}" not found` - ); + .returning(); + + if (stepValue) { + step = deserializeStepError(compact(stepValue)); + } else { + const [existing] = await tx + .select({ status: Schema.steps.status }) + .from(Schema.steps) + .where( + and( + eq(Schema.steps.runId, effectiveRunId), + eq(Schema.steps.stepId, data.correlationId!) + ) + ) + .limit(1); + if (!existing) { + throw new WorkflowWorldError( + `Step "${data.correlationId}" not found` + ); + } + if (isTerminalStepStatus(existing.status)) { + throw new EntityConflictError( + `Cannot modify step in terminal state "${existing.status}"` + ); + } } - if (isTerminalStepStatus(existing.status)) { + + // A ULID-numbered step_started takes its id only after the guarded + // step UPDATE has acquired and passed the row lock. Without a + // sequence, this is the local ordering guarantee we can provide: a + // writer blocked on the step row will not carry an older event id + // into a later insert. A slot id is exempt — it names a position in + // the log, not a time — and the caller may have claimed it already. + const stepStartedEventId = ids.primary(); + eventId = stepStartedEventId; + const [eventValue] = await tx + .insert(events) + .values({ + runId: effectiveRunId, + eventId: stepStartedEventId, + correlationId: data.correlationId, + eventType: data.eventType, + eventData: storedEventData, + specVersion: effectiveSpecVersion, + }) + .returning({ createdAt: events.createdAt }); + + if (!eventValue) { throw new EntityConflictError( - `Cannot modify step in terminal state "${existing.status}"` + `Event ${stepStartedEventId} could not be created` ); } - } - - // Allocate the step_started ULID only after the guarded step UPDATE - // has acquired and passed the row lock. Without a sequence, this is - // the local ordering guarantee we can provide: a writer blocked on - // the step row will not carry an older event id into a later insert. - const stepStartedEventId = `wevt_${ulid()}`; - eventId = stepStartedEventId; - const [eventValue] = await tx - .insert(events) - .values({ - runId: effectiveRunId, - eventId: stepStartedEventId, - correlationId: data.correlationId, - eventType: data.eventType, - eventData: storedEventData, - specVersion: effectiveSpecVersion, - }) - .returning({ createdAt: events.createdAt }); - - if (!eventValue) { - throw new EntityConflictError( - `Event ${stepStartedEventId} could not be created` - ); - } - return eventValue; - }); + return eventValue; + }) + ); } // Handle step_completed event: update step status @@ -1531,20 +1727,21 @@ export function createEventsStorage(drizzle: Drizzle): Storage['events'] { token: eventData.token, conflictingRunId: existingHook.runId, }; + const [conflictValue] = await publish((ids) => + drizzle + .insert(events) + .values({ + runId: effectiveRunId, + eventId: ids.primary(), + correlationId: data.correlationId, + eventType: 'hook_conflict', + eventData: conflictEventData, + specVersion: effectiveSpecVersion, + }) + .returning({ createdAt: events.createdAt }) + ); const conflictEventId = getEventId(); - const [conflictValue] = await drizzle - .insert(events) - .values({ - runId: effectiveRunId, - eventId: conflictEventId, - correlationId: data.correlationId, - eventType: 'hook_conflict', - eventData: conflictEventData, - specVersion: effectiveSpecVersion, - }) - .returning({ createdAt: events.createdAt }); - if (!conflictValue) { throw new EntityConflictError( `Event ${conflictEventId} could not be created` @@ -1622,47 +1819,49 @@ export function createEventsStorage(drizzle: Drizzle): Storage['events'] { // guarded UPDATE linearizes against a concurrent terminal step // event. if (data.eventType === 'hook_received') { - value = await drizzle.transaction(async (tx) => { - const [runRow] = await tx - .select({ status: Schema.runs.status }) - .from(Schema.runs) - .where(eq(Schema.runs.runId, effectiveRunId)) - .for('update') - .limit(1); - if (!runRow) { - throw new WorkflowRunNotFoundError(effectiveRunId); - } - if (isTerminalWorkflowRunStatus(runRow.status)) { - throw new RunExpiredError( - `Workflow run "${effectiveRunId}" is already in terminal state "${runRow.status}"` - ); - } + value = await publish((ids) => + drizzle.transaction(async (tx) => { + const [runRow] = await tx + .select({ status: Schema.runs.status }) + .from(Schema.runs) + .where(eq(Schema.runs.runId, effectiveRunId)) + .for('update') + .limit(1); + if (!runRow) { + throw new WorkflowRunNotFoundError(effectiveRunId); + } + if (isTerminalWorkflowRunStatus(runRow.status)) { + throw new RunExpiredError( + `Workflow run "${effectiveRunId}" is already in terminal state "${runRow.status}"` + ); + } - // Allocate the ULID only after the row lock is acquired, - // matching step_started's ordering guarantee: a writer blocked - // on the run row must not carry an older event id into a later - // insert. - const hookReceivedEventId = `wevt_${ulid()}`; - eventId = hookReceivedEventId; - const [eventValue] = await tx - .insert(events) - .values({ - runId: effectiveRunId, - eventId: hookReceivedEventId, - correlationId: data.correlationId, - eventType: data.eventType, - eventData: storedEventData, - specVersion: effectiveSpecVersion, - }) - .returning({ createdAt: events.createdAt }); + // Take the ULID only after the row lock is acquired, matching + // step_started's ordering guarantee: a writer blocked on the run + // row must not carry an older event id into a later insert. A slot + // id names a position rather than a time, so it is exempt. + const hookReceivedEventId = ids.primary(); + eventId = hookReceivedEventId; + const [eventValue] = await tx + .insert(events) + .values({ + runId: effectiveRunId, + eventId: hookReceivedEventId, + correlationId: data.correlationId, + eventType: data.eventType, + eventData: storedEventData, + specVersion: effectiveSpecVersion, + }) + .returning({ createdAt: events.createdAt }); - if (!eventValue) { - throw new EntityConflictError( - `Event ${hookReceivedEventId} could not be created` - ); - } - return eventValue; - }); + if (!eventValue) { + throw new EntityConflictError( + `Event ${hookReceivedEventId} could not be created` + ); + } + return eventValue; + }) + ); } // Handle wait_created event: create wait entity @@ -1748,17 +1947,23 @@ export function createEventsStorage(drizzle: Drizzle): Storage['events'] { try { if (!value) { - [value] = await drizzle - .insert(events) - .values({ - runId: effectiveRunId, - eventId: getEventId(), - correlationId: data.correlationId, - eventType: data.eventType, - eventData: storedEventData, - specVersion: effectiveSpecVersion, - }) - .returning({ createdAt: events.createdAt }); + // Only the event row is retried here: the entity this event describes + // was materialized above, outside any transaction, and re-inserting + // the event at a higher position leaves the log dense and still + // consistent with that entity. + [value] = await publish((ids) => + drizzle + .insert(events) + .values({ + runId: effectiveRunId, + eventId: ids.primary(), + correlationId: data.correlationId, + eventType: data.eventType, + eventData: storedEventData, + specVersion: effectiveSpecVersion, + }) + .returning({ createdAt: events.createdAt }) + ); } } catch (err) { // Translate unique-violation on the correlated-event partial index diff --git a/packages/world-postgres/test/slot-identity.test.ts b/packages/world-postgres/test/slot-identity.test.ts new file mode 100644 index 0000000000..d8b0578a7d --- /dev/null +++ b/packages/world-postgres/test/slot-identity.test.ts @@ -0,0 +1,375 @@ +import { execSync } from 'node:child_process'; +import { PostgreSqlContainer } from '@testcontainers/postgresql'; +import { SlotConflictError } from '@workflow/errors'; +import { + FIRST_SLOT, + maxSlotOf, + SPEC_VERSION_CURRENT, + SPEC_VERSION_SLOT_IDENTITY, + slotEventId, + slotFromId, +} from '@workflow/world'; +import { Pool } from 'pg'; +import { + afterAll, + beforeAll, + beforeEach, + describe, + expect, + test, +} from 'vitest'; +import { createClient } from '../src/drizzle/index.js'; +import { createEventsStorage } from '../src/storage.js'; + +describe('Slot identity (Postgres integration)', () => { + if (process.platform === 'win32') { + test.skip('skipped on Windows since it relies on a docker container', () => {}); + return; + } + + let container: Awaited>; + let pool: Pool; + let events: ReturnType; + + beforeAll(async () => { + container = await new PostgreSqlContainer('postgres:15-alpine').start(); + const dbUrl = container.getConnectionUri(); + process.env.DATABASE_URL = dbUrl; + process.env.WORKFLOW_POSTGRES_URL = dbUrl; + execSync('pnpm db:push', { + stdio: 'inherit', + cwd: process.cwd(), + env: process.env, + }); + // Contention is the point of these tests, so the pool has to be able to + // hold every writer of a burst at once. + pool = new Pool({ connectionString: dbUrl, max: 20 }); + events = createEventsStorage(createClient(pool)); + }, 120_000); + + beforeEach(async () => { + await pool.query( + 'TRUNCATE TABLE workflow.workflow_events, workflow.workflow_steps, workflow.workflow_hooks, workflow.workflow_runs RESTART IDENTITY CASCADE' + ); + }); + + afterAll(async () => { + await pool.end(); + await container.stop(); + }); + + /** Start a run whose events are numbered by slot, and return its id. */ + async function newSlotRun(): Promise { + const result = await events.create(null, { + eventType: 'run_created', + specVersion: SPEC_VERSION_SLOT_IDENTITY, + eventData: { + deploymentId: 'dpl_test', + workflowName: 'test-workflow', + input: new Uint8Array(), + }, + }); + if (!result.run) { + throw new Error('Expected run to be created'); + } + return result.run.runId; + } + + function eventsOf(runId: string) { + // The page size is explicit: the default would silently truncate a fan-out + // and make a dense log look sparse. + return events.list({ runId, pagination: { limit: 500 } }); + } + + /** The slots of the run's log, in list order. */ + async function slotsOf(runId: string): Promise { + const { data } = await eventsOf(runId); + return data.map((event) => slotFromId(event.eventId) ?? -1); + } + + function ascending(slots: number[]): number[] { + return [...slots].sort((a, b) => a - b); + } + + function denseFrom(count: number): number[] { + return Array.from({ length: count }, (_, index) => FIRST_SLOT + index); + } + + async function createStep( + runId: string, + stepId: string, + eventId?: string + ): Promise { + const result = await events.create( + runId, + { + eventType: 'step_created', + specVersion: SPEC_VERSION_SLOT_IDENTITY, + correlationId: stepId, + eventData: { stepName: 'a-step', input: new Uint8Array() }, + }, + eventId === undefined ? undefined : { eventId } + ); + if (!result.event) { + throw new Error('Expected an event'); + } + return result.event.eventId; + } + + describe('numbering', () => { + test('puts run_created in the first slot', async () => { + const runId = await newSlotRun(); + await expect(slotsOf(runId)).resolves.toEqual([FIRST_SLOT]); + }); + + test('allocates dense slots for writers that hold no log', async () => { + // A step completion reporting in, a cancellation from an API call: the + // caller has no event log, so the world numbers the event for it. + const runId = await newSlotRun(); + await createStep(runId, 'step_a'); + await createStep(runId, 'step_b'); + await expect(slotsOf(runId)).resolves.toEqual(denseFrom(3)); + }); + + test('honours a slot the caller claims', async () => { + const runId = await newSlotRun(); + const eventId = await createStep(runId, 'step_a', slotEventId(2)); + expect(eventId).toBe(slotEventId(2)); + await expect(slotsOf(runId)).resolves.toEqual(denseFrom(2)); + }); + + test('numbers a ULID-mode run the way it always did', async () => { + const created = await events.create(null, { + eventType: 'run_created', + specVersion: SPEC_VERSION_CURRENT, + eventData: { + deploymentId: 'dpl_test', + workflowName: 'test-workflow', + input: new Uint8Array(), + }, + }); + const runId = created.run?.runId as string; + const eventId = await createStep(runId, 'step_a'); + expect(eventId).toMatch(/^wevt_/); + expect(slotFromId(eventId)).toBeUndefined(); + }); + + test('gives a lazy step start two consecutive slots', async () => { + // One request, two events: the step_started the caller sent and the + // step_created it deferred. Which sorts first does not matter — only + // step_created flips the client's hasCreatedEvent — but both have to + // land, and neither may leave a hole. + const runId = await newSlotRun(); + const started = await events.create(runId, { + eventType: 'step_started', + specVersion: SPEC_VERSION_SLOT_IDENTITY, + correlationId: 'step_a', + eventData: { stepName: 'a-step', input: new Uint8Array() }, + }); + expect(slotFromId(started.event?.eventId ?? '')).toBe(2); + const { data } = await eventsOf(runId); + expect( + data.map((event) => `${slotFromId(event.eventId)} ${event.eventType}`) + ).toEqual(['1 run_created', '2 step_started', '3 step_created']); + }); + + test('allocates the deferred step_created above a claimed slot', async () => { + // A caller that defers a step_created cannot know it will be synthesized, + // so it never claims a slot for it — and the slot it did claim is its own. + const runId = await newSlotRun(); + const started = await events.create( + runId, + { + eventType: 'step_started', + specVersion: SPEC_VERSION_SLOT_IDENTITY, + correlationId: 'step_a', + eventData: { stepName: 'a-step', input: new Uint8Array() }, + }, + { eventId: slotEventId(2) } + ); + expect(started.event?.eventId).toBe(slotEventId(2)); + expect(ascending(await slotsOf(runId))).toEqual(denseFrom(3)); + }); + + test('numbers events of runs it never created', async () => { + // `step_completed` and `step_retrying` deliberately skip the run read, so + // the mode comes from the log rather than from a run row in hand. + const runId = await newSlotRun(); + await createStep(runId, 'step_a'); + const completed = await events.create(runId, { + eventType: 'step_completed', + specVersion: SPEC_VERSION_CURRENT, + correlationId: 'step_a', + eventData: { output: new Uint8Array() }, + }); + expect(slotFromId(completed.event?.eventId ?? '')).toBe(3); + }); + }); + + describe('contention', () => { + // The primary key is the authority, and a writer that loses a position is + // retried at one that is still free. Nothing here may leave a hole: density + // is what lets a reader prove its copy of a log is complete. + for (const writers of [2, 8, 50]) { + test(`keeps ${writers} concurrent writers dense`, async () => { + const runId = await newSlotRun(); + const ids = await Promise.all( + Array.from({ length: writers }, (_, index) => + createStep(runId, `step_${index}`) + ) + ); + expect(new Set(ids).size).toBe(writers); + expect(ascending(await slotsOf(runId))).toEqual(denseFrom(writers + 1)); + }, 60_000); + } + + test('proves completeness: the highest slot is the event count', async () => { + const runId = await newSlotRun(); + await Promise.all( + Array.from({ length: 5 }, (_, index) => + createStep(runId, `step_${index}`) + ) + ); + const { data } = await eventsOf(runId); + expect(maxSlotOf(data)).toBe(data.length); + }); + + test('leaves no hole behind a rejected write', async () => { + // The rejected op's slot sits below its concurrent sibling's, and a hole + // below a published event can never be filled. + const runId = await newSlotRun(); + const [rejected, accepted] = await Promise.allSettled([ + events.create(runId, { + eventType: 'step_completed', + specVersion: SPEC_VERSION_SLOT_IDENTITY, + correlationId: 'step_never_created', + eventData: { output: new Uint8Array() }, + }), + createStep(runId, 'step_a'), + ]); + expect(rejected.status).toBe('rejected'); + expect(accepted.status).toBe('fulfilled'); + await createStep(runId, 'step_b'); + expect(ascending(await slotsOf(runId))).toEqual(denseFrom(3)); + }); + }); + + describe('mode is pinned to the run', () => { + test('rejects a slot id claimed on a ULID-numbered run', async () => { + const created = await events.create(null, { + eventType: 'run_created', + specVersion: SPEC_VERSION_CURRENT, + eventData: { + deploymentId: 'dpl_test', + workflowName: 'test-workflow', + input: new Uint8Array(), + }, + }); + const runId = created.run?.runId as string; + await expect(createStep(runId, 'step_a', slotEventId(2))).rejects.toThrow( + /not numbered by slot/ + ); + }); + + test('rejects a ULID id claimed on a slot-numbered run', async () => { + const runId = await newSlotRun(); + await expect( + createStep(runId, 'step_a', 'evnt_01K5Z0000000000000000000AA') + ).rejects.toThrow(/not a slot id/); + }); + + test('ignores the spec version of later requests', async () => { + // A run is in exactly one mode for life; only what was persisted decides. + const runId = await newSlotRun(); + const result = await events.create(runId, { + eventType: 'step_created', + specVersion: SPEC_VERSION_CURRENT, + correlationId: 'step_a', + eventData: { stepName: 'a-step', input: new Uint8Array() }, + }); + expect(slotFromId(result.event?.eventId ?? '')).toBe(2); + }); + }); + + describe('conflict', () => { + test('reports the events the loser is missing', async () => { + const runId = await newSlotRun(); + // Out of band: something else takes the slot this caller was about to + // claim, so the caller's log is provably missing an event. + await createStep(runId, 'step_out_of_band'); + + const conflict = await events + .create( + runId, + { + eventType: 'step_created', + specVersion: SPEC_VERSION_SLOT_IDENTITY, + correlationId: 'step_a', + eventData: { stepName: 'a-step', input: new Uint8Array() }, + }, + { eventId: slotEventId(2), maxSlot: 1 } + ) + .catch((error: unknown) => error); + + expect(SlotConflictError.is(conflict)).toBe(true); + const slotConflict = conflict as SlotConflictError; + expect(slotConflict.status).toBe(409); + expect(slotConflict.eventId).toBe(slotEventId(2)); + expect(slotConflict.events?.map((event) => event.eventId)).toEqual([ + slotEventId(2), + ]); + }); + + test('excludes events the loser already holds from the delta', async () => { + const runId = await newSlotRun(); + await createStep(runId, 'step_one'); + await createStep(runId, 'step_two'); + + const conflict = await events + .create( + runId, + { + eventType: 'step_created', + specVersion: SPEC_VERSION_SLOT_IDENTITY, + correlationId: 'step_a', + eventData: { stepName: 'a-step', input: new Uint8Array() }, + }, + { eventId: slotEventId(2), maxSlot: 2 } + ) + .catch((error: unknown) => error); + + // Slots 1 and 2 are at or below what the caller had; only 3 is news. + expect( + (conflict as SlotConflictError).events?.map((event) => event.eventId) + ).toEqual([slotEventId(3)]); + }); + + test('lets the loser re-propose at the next free slot', async () => { + const runId = await newSlotRun(); + await createStep(runId, 'step_out_of_band'); + await expect(createStep(runId, 'step_a', slotEventId(2))).rejects.toThrow( + SlotConflictError + ); + // Merging the delta moves the caller's own numbering forward by one. + const eventId = await createStep(runId, 'step_a', slotEventId(3)); + expect(eventId).toBe(slotEventId(3)); + await expect(slotsOf(runId)).resolves.toEqual(denseFrom(3)); + }); + + test('materializes nothing for a claim that is already taken', async () => { + // The re-post is what the guard protects: a step row left behind by the + // losing attempt would make the retry trip its own orphan and read that + // as "a concurrent handler won the create". + const runId = await newSlotRun(); + await createStep(runId, 'step_out_of_band'); + await expect(createStep(runId, 'step_a', slotEventId(2))).rejects.toThrow( + SlotConflictError + ); + const { rows } = await pool.query( + 'SELECT step_id FROM workflow.workflow_steps WHERE run_id = $1', + [runId] + ); + expect(rows.map((row) => row.step_id)).toEqual(['step_out_of_band']); + }); + }); +}); diff --git a/packages/world/src/index.ts b/packages/world/src/index.ts index 7317b340ba..5a8c185f2a 100644 --- a/packages/world/src/index.ts +++ b/packages/world/src/index.ts @@ -113,9 +113,12 @@ export { export type { SpecVersion } from './spec-version.js'; export { isLegacySpecVersion, + mintedSpecVersion, requiresNewerWorld, + SLOT_IDENTITY_ENV_VAR, SPEC_VERSION_CURRENT, SPEC_VERSION_LEGACY, + SPEC_VERSION_MAX_SUPPORTED, SPEC_VERSION_SLOT_IDENTITY, SPEC_VERSION_SUPPORTS_ATTRIBUTES, SPEC_VERSION_SUPPORTS_CBOR_QUEUE_TRANSPORT, diff --git a/packages/world/src/slot-identity.test.ts b/packages/world/src/slot-identity.test.ts index c99d2aa930..acdfada627 100644 --- a/packages/world/src/slot-identity.test.ts +++ b/packages/world/src/slot-identity.test.ts @@ -1,5 +1,6 @@ import { ulid } from 'ulid'; import { describe, expect, it } from 'vitest'; +import { z } from 'zod'; import { FIRST_SLOT, isSlotId, @@ -17,9 +18,9 @@ describe('slotIdBody', () => { expect(body).toHaveLength(SLOT_ID_WIDTH); expect(`evnt_${body}`).toHaveLength(`evnt_${ulid()}`.length); // Crockford base32 starts with the decimal digits, so the padded body - // parses as a ULID — this is what keeps every existing schema, sort key and - // range fence working unchanged. - expect(ulidToDate(body)).not.toBeNull(); + // satisfies the ULID syntax — this is what keeps every existing schema, + // sort key and range fence working unchanged. + expect(z.string().ulid().safeParse(body).success).toBe(true); }); it('orders lexicographically by slot at a fixed width', () => { @@ -60,6 +61,24 @@ describe('slotFromId', () => { }); }); +describe('a slot carries no timestamp', () => { + it('reports no time rather than epoch 0', () => { + // Passing the ULID syntax check is what makes a slot portable; decoding a + // *time* out of one is always a bug. Two that this guards: the sandbox + // clock is set from the events it consumes, so epoch 0 would rewind a + // replaying workflow's `Date.now()` to 1970; and world-local prefilters + // cursor pagination on the time in the filename, so epoch 0 would hide + // every slot-numbered event from an ascending page. + expect(ulidToDate(slotIdBody(FIRST_SLOT))).toBeNull(); + expect(ulidToDate(slotEventId(FIRST_SLOT))).toBeNull(); + expect(ulidToDate(slotIdBody(123_456))).toBeNull(); + }); + + it('still reads the time out of a ULID', () => { + expect(ulidToDate(ulid())?.getTime()).toBeGreaterThan(0); + }); +}); + describe('maxSlotOf', () => { it('finds the highest slot regardless of position', () => { // A log is merged from several loads and is not sorted, so the last element diff --git a/packages/world/src/spec-version.test.ts b/packages/world/src/spec-version.test.ts index 880c43180d..f21596f951 100644 --- a/packages/world/src/spec-version.test.ts +++ b/packages/world/src/spec-version.test.ts @@ -1,9 +1,13 @@ import { describe, expect, it } from 'vitest'; import { isLegacySpecVersion, + mintedSpecVersion, requiresNewerWorld, + SLOT_IDENTITY_ENV_VAR, SPEC_VERSION_CURRENT, SPEC_VERSION_LEGACY, + SPEC_VERSION_MAX_SUPPORTED, + SPEC_VERSION_SLOT_IDENTITY, SPEC_VERSION_SUPPORTS_ATTRIBUTES, SPEC_VERSION_SUPPORTS_COMPRESSION, } from './spec-version.js'; @@ -13,10 +17,21 @@ describe('spec version constants', () => { expect(SPEC_VERSION_CURRENT).toBe(SPEC_VERSION_SUPPORTS_COMPRESSION); expect(SPEC_VERSION_SUPPORTS_COMPRESSION).toBe(5); }); + + it('can read a newer spec version than it mints', () => { + // Slot identity is readable by every world before any world mints it, so + // that turning it on for new runs cannot make those same worlds reject + // them. Once slots are the default the two constants coincide again. + expect(SPEC_VERSION_MAX_SUPPORTED).toBe(SPEC_VERSION_SLOT_IDENTITY); + expect(SPEC_VERSION_MAX_SUPPORTED).toBeGreaterThanOrEqual( + SPEC_VERSION_CURRENT + ); + }); }); describe('requiresNewerWorld', () => { - it('accepts runs at or below the current spec version', () => { + it('accepts runs at or below the newest readable spec version', () => { + expect(requiresNewerWorld(SPEC_VERSION_MAX_SUPPORTED)).toBe(false); expect(requiresNewerWorld(SPEC_VERSION_CURRENT)).toBe(false); expect(requiresNewerWorld(SPEC_VERSION_SUPPORTS_ATTRIBUTES)).toBe(false); expect(requiresNewerWorld(SPEC_VERSION_LEGACY)).toBe(false); @@ -24,13 +39,20 @@ describe('requiresNewerWorld', () => { expect(requiresNewerWorld(null)).toBe(false); }); - it('rejects runs newer than the current spec version', () => { + it('accepts a slot-identity run', () => { + // Gates the flag rollout: a world that rejected spec-6 would reject the + // runs it had just stamped spec-6 itself, at their first event after + // run_created. + expect(requiresNewerWorld(SPEC_VERSION_SLOT_IDENTITY)).toBe(false); + }); + + it('rejects runs newer than the newest readable spec version', () => { // This is the contract that protects older SDKs from compressed // payloads they cannot decode: a spec-5 run read by an SDK whose - // SPEC_VERSION_CURRENT is 4 fails this check up front (with - // RunNotSupportedError at the storage layer) instead of failing on - // individual compressed payloads. - expect(requiresNewerWorld(SPEC_VERSION_CURRENT + 1)).toBe(true); + // ceiling is 4 fails this check up front (with RunNotSupportedError at + // the storage layer) instead of failing on individual compressed + // payloads. + expect(requiresNewerWorld(SPEC_VERSION_MAX_SUPPORTED + 1)).toBe(true); }); it('simulates a v4 reader rejecting a compression-era run', () => { @@ -51,3 +73,33 @@ describe('isLegacySpecVersion', () => { expect(isLegacySpecVersion(5)).toBe(false); }); }); + +describe('mintedSpecVersion', () => { + it('mints the current version by default', () => { + expect(mintedSpecVersion({})).toBe(SPEC_VERSION_CURRENT); + }); + + it('mints slot identity when the flag is set', () => { + for (const value of ['1', 'true']) { + expect(mintedSpecVersion({ [SLOT_IDENTITY_ENV_VAR]: value })).toBe( + SPEC_VERSION_SLOT_IDENTITY + ); + } + }); + + it('treats any other value as off', () => { + // An unset-but-present variable is the shape a shell leaves behind, and it + // must not silently switch a deployment's event identity scheme. + for (const value of ['', '0', 'false', 'yes']) { + expect(mintedSpecVersion({ [SLOT_IDENTITY_ENV_VAR]: value })).toBe( + SPEC_VERSION_CURRENT + ); + } + }); + + it('mints nothing a world cannot read', () => { + expect( + requiresNewerWorld(mintedSpecVersion({ [SLOT_IDENTITY_ENV_VAR]: '1' })) + ).toBe(false); + }); +}); diff --git a/packages/world/src/spec-version.ts b/packages/world/src/spec-version.ts index d9be743be6..a86f4f3e90 100644 --- a/packages/world/src/spec-version.ts +++ b/packages/world/src/spec-version.ts @@ -42,20 +42,60 @@ export const SPEC_VERSION_SUPPORTS_COMPRESSION = 5 as SpecVersion; * persisted `specVersion`, never from the build. A run started under ULID * correlation ids and replayed by a slot-capable build would otherwise propose * `step_…001` where its log holds `step_01K…`, matching no existing entity. - * - * Deliberately not `SPEC_VERSION_CURRENT` yet: `requiresNewerWorld()` is what - * makes a world reject runs it cannot read, so bumping current before the - * worlds can allocate slots would have them reject their own new runs. */ export const SPEC_VERSION_SLOT_IDENTITY = 6 as SpecVersion; /** * Current spec version (event-sourced architecture with native attributes * and compressed payloads). + * + * This is the version new runs are stamped with, which is a *lower* bar than + * the newest version this build can read — see + * {@link SPEC_VERSION_MAX_SUPPORTED}. Slot identity ships behind a flag, so it + * is readable everywhere before it is minted anywhere. */ export const SPEC_VERSION_CURRENT = SPEC_VERSION_SUPPORTS_COMPRESSION as SpecVersion; +/** + * Newest spec version this build can read. Runs above it are rejected outright + * by {@link requiresNewerWorld} rather than misread. + * + * Distinct from {@link SPEC_VERSION_CURRENT} because a world has to be able to + * read a version before anything may mint it: the flag that turns slot identity + * on for new runs would otherwise make every world reject the runs it had just + * created. Worlds opt into minting individually, via the `specVersion` they + * declare. + */ +export const SPEC_VERSION_MAX_SUPPORTED = + SPEC_VERSION_SLOT_IDENTITY as SpecVersion; + +/** + * Environment variable that opts new runs into slot identity. + * + * Read per `createWorld()` call rather than at module load, so a test or a + * single process can create worlds in both modes. + */ +export const SLOT_IDENTITY_ENV_VAR = 'WORKFLOW_SLOT_IDENTITY'; + +/** + * The spec version a world should stamp on the runs it creates: slot identity + * when {@link SLOT_IDENTITY_ENV_VAR} is set, otherwise + * {@link SPEC_VERSION_CURRENT}. + * + * Every world reads runs up to {@link SPEC_VERSION_MAX_SUPPORTED} whatever this + * returns, so turning the flag on in one place does not make the runs it creates + * unreadable elsewhere. + */ +export function mintedSpecVersion( + env: Record = process.env +): SpecVersion { + const value = env[SLOT_IDENTITY_ENV_VAR]; + return value === '1' || value === 'true' + ? SPEC_VERSION_SLOT_IDENTITY + : SPEC_VERSION_CURRENT; +} + /** * Check if a spec version is legacy (<= SPEC_VERSION_LEGACY or undefined). * Legacy runs require different handling - they use direct entity mutation @@ -73,7 +113,7 @@ export function isLegacySpecVersion(v: number | undefined | null): boolean { } /** - * Check if a spec version requires a newer world (> SPEC_VERSION_CURRENT). + * Check if a spec version requires a newer world (> SPEC_VERSION_MAX_SUPPORTED). * This happens when a run was created by a newer SDK version. * * @param v - The spec version number, or undefined/null for legacy runs @@ -81,7 +121,7 @@ export function isLegacySpecVersion(v: number | undefined | null): boolean { */ export function requiresNewerWorld(v: number | undefined | null): boolean { if (v === undefined || v === null) return false; - return v > SPEC_VERSION_CURRENT; + return v > SPEC_VERSION_MAX_SUPPORTED; } /** diff --git a/packages/world/src/ulid.ts b/packages/world/src/ulid.ts index 1ee1b7b47c..cc893834c3 100644 --- a/packages/world/src/ulid.ts +++ b/packages/world/src/ulid.ts @@ -1,5 +1,6 @@ import { decodeTime } from 'ulid'; import { z } from 'zod'; +import { isSlotId } from './slot-identity.js'; const UlidSchema = z.string().ulid(); @@ -36,8 +37,19 @@ export const DEFAULT_TIMESTAMP_THRESHOLD_MS = /** * Extracts a Date from a ULID string, or null if the string is not a valid ULID. + * + * Slot ids are not ULIDs even though they pass the ULID *syntax* check: their + * body is all decimal digits, which Crockford base32 accepts, and it would + * decode to a timestamp of epoch 0 instead of failing. A slot encodes a + * position, not a time, so it is reported here as having no time at all — + * callers must read the object's own `createdAt`. Silently returning 1970 + * instead would, among other things, rewind a replaying workflow's clock and + * make cursor pagination skip every slot-numbered event. */ export function ulidToDate(maybeUlid: string): Date | null { + if (isSlotId(maybeUlid)) { + return null; + } const ulid = UlidSchema.safeParse(maybeUlid); if (!ulid.success) { return null; From e5af056d607e539a7122ce6442c7ee7c8b08f0ff Mon Sep 17 00:00:00 2001 From: Peter Wielander Date: Thu, 30 Jul 2026 21:48:59 -0700 Subject: [PATCH 10/35] feat(worlds): mint slot identity by default, including from world-vercel MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit `mintedSpecVersion()` now returns spec 6 unless `WORKFLOW_SLOT_IDENTITY` is explicitly `0`/`false`, and world-vercel reads it instead of hardcoding spec 5. world-vercel was the one World that never called it, so a deployment could not mint a slot-numbered run however the flag was set — the flag only reached the Local and Postgres Worlds. Every World still reads both schemes, and mode stays pinned to a run's persisted specVersion, so flipping the default only changes how runs created from here on are numbered. --- .changeset/slot-identity-default-on.md | 8 +++++ .../docs/v5/configuration/runtime-tuning.mdx | 6 ++-- packages/world-local/src/index.ts | 6 ++-- packages/world-postgres/src/index.ts | 6 ++-- packages/world-vercel/src/index.ts | 11 ++++--- packages/world/src/spec-version.test.ts | 23 +++++++-------- packages/world/src/spec-version.ts | 29 +++++++++---------- 7 files changed, 49 insertions(+), 40 deletions(-) create mode 100644 .changeset/slot-identity-default-on.md diff --git a/.changeset/slot-identity-default-on.md b/.changeset/slot-identity-default-on.md new file mode 100644 index 0000000000..55e3973586 --- /dev/null +++ b/.changeset/slot-identity-default-on.md @@ -0,0 +1,8 @@ +--- +'@workflow/world': minor +'@workflow/world-vercel': minor +'@workflow/world-local': minor +'@workflow/world-postgres': minor +--- + +Number new runs' events by position by default, in every World including the Vercel one. Set `WORKFLOW_SLOT_IDENTITY=0` to keep minting ULID event ids. diff --git a/docs/content/docs/v5/configuration/runtime-tuning.mdx b/docs/content/docs/v5/configuration/runtime-tuning.mdx index 110e569d5f..ba4a05b397 100644 --- a/docs/content/docs/v5/configuration/runtime-tuning.mdx +++ b/docs/content/docs/v5/configuration/runtime-tuning.mdx @@ -50,12 +50,12 @@ For example, a workflow can run a 10-minute inline step even with `WORKFLOW_REPL ### `WORKFLOW_SLOT_IDENTITY` -- Default: disabled +- Default: enabled - Numbers a new run's events by position instead of by ULID: `evnt_…001` is the run's first event, `evnt_…002` its second, with no gaps. A reader can then prove its copy of the log is complete, because the highest number is the event count. - Contention becomes explicit rather than silent. Two writers proposing the same position cannot both win: the loser gets a 409 ([`SlotConflictError`](/docs/api-reference/workflow-errors/slot-conflict-error)) carrying the events it was missing, and the runtime merges them, replays, and re-proposes at the next free position. -- Applies only to runs created while it is set. A run keeps the identity scheme it was created with for life, so turning the flag on or off never affects runs already in flight. +- Applies only to runs created while it is enabled. A run keeps the identity scheme it was created with for life, so turning the flag on or off never affects runs already in flight. - Requires a World that supports it. A World that does not rejects the run outright rather than mis-numbering its events. -- Set `1` or `true` to enable. +- Set `0` or `false` to disable, which numbers new runs by ULID as before. ## Inline execution diff --git a/packages/world-local/src/index.ts b/packages/world-local/src/index.ts index 2014a75652..aa77546732 100644 --- a/packages/world-local/src/index.ts +++ b/packages/world-local/src/index.ts @@ -72,9 +72,9 @@ export function createWorld(args?: Partial): LocalWorld { ); const recoverActiveRuns = resolveRecoverActiveRuns(mergedConfig); return { - // What this world stamps on new runs, which is not the newest version it - // can read: slot identity is readable everywhere and minted only where - // WORKFLOW_SLOT_IDENTITY is set. + // What this world stamps on new runs: slot identity, unless + // WORKFLOW_SLOT_IDENTITY switches it off. Every world reads both schemes + // whatever this says. specVersion: mintedSpecVersion(), ...queue, ...storage, diff --git a/packages/world-postgres/src/index.ts b/packages/world-postgres/src/index.ts index 0db47b89e2..926d752d3b 100644 --- a/packages/world-postgres/src/index.ts +++ b/packages/world-postgres/src/index.ts @@ -63,9 +63,9 @@ export function createWorld( const streamer = createStreamer(pool, drizzle); return { - // What this world stamps on new runs, which is not the newest version it - // can read: slot identity is readable everywhere and minted only where - // WORKFLOW_SLOT_IDENTITY is set. + // What this world stamps on new runs: slot identity, unless + // WORKFLOW_SLOT_IDENTITY switches it off. Every world reads both schemes + // whatever this says. specVersion: mintedSpecVersion(), ...storage, ...streamer, diff --git a/packages/world-vercel/src/index.ts b/packages/world-vercel/src/index.ts index 42ff155fda..6a68e6845a 100644 --- a/packages/world-vercel/src/index.ts +++ b/packages/world-vercel/src/index.ts @@ -1,5 +1,5 @@ import type { World } from '@workflow/world'; -import { SPEC_VERSION_SUPPORTS_COMPRESSION } from '@workflow/world'; +import { mintedSpecVersion } from '@workflow/world'; import { createAnalytics } from './analytics.js'; import { createRunId, describeRun } from './create-run-id.js'; import { createGetEncryptionKeyForRun } from './encryption.js'; @@ -29,9 +29,12 @@ export function createWorld(config?: APIConfig): World { config?.projectConfig?.projectId || process.env.VERCEL_PROJECT_ID; return { - // Spec v5 adds client-side zstd/gzip payload compression. The server stores - // those payloads opaquely, and v5 remains a superset of v4 attributes. - specVersion: SPEC_VERSION_SUPPORTS_COMPRESSION, + // What this world stamps on new runs: slot identity (spec v6) unless + // WORKFLOW_SLOT_IDENTITY switches it off, in which case v5 — client-side + // zstd/gzip payload compression over a superset of the v4 attributes. + // Either way this world reads both, so the stamp only decides how the runs + // it creates from here on are numbered. + specVersion: mintedSpecVersion(), capabilities: { // workflow-server enforces the `stateUpdatedAt` optimistic-concurrency // guard: creations carrying a stale snapshot are rejected with 412 diff --git a/packages/world/src/spec-version.test.ts b/packages/world/src/spec-version.test.ts index f21596f951..0125e2fb1c 100644 --- a/packages/world/src/spec-version.test.ts +++ b/packages/world/src/spec-version.test.ts @@ -75,31 +75,30 @@ describe('isLegacySpecVersion', () => { }); describe('mintedSpecVersion', () => { - it('mints the current version by default', () => { - expect(mintedSpecVersion({})).toBe(SPEC_VERSION_CURRENT); + it('mints slot identity by default', () => { + expect(mintedSpecVersion({})).toBe(SPEC_VERSION_SLOT_IDENTITY); }); - it('mints slot identity when the flag is set', () => { - for (const value of ['1', 'true']) { + it('mints the previous version when the flag is switched off', () => { + for (const value of ['0', 'false']) { expect(mintedSpecVersion({ [SLOT_IDENTITY_ENV_VAR]: value })).toBe( - SPEC_VERSION_SLOT_IDENTITY + SPEC_VERSION_CURRENT ); } }); - it('treats any other value as off', () => { + it('treats any other value as on', () => { // An unset-but-present variable is the shape a shell leaves behind, and it - // must not silently switch a deployment's event identity scheme. - for (const value of ['', '0', 'false', 'yes']) { + // must not silently switch a deployment's event identity scheme. Opting + // out takes an explicit `0`/`false`. + for (const value of ['', '1', 'true', 'yes']) { expect(mintedSpecVersion({ [SLOT_IDENTITY_ENV_VAR]: value })).toBe( - SPEC_VERSION_CURRENT + SPEC_VERSION_SLOT_IDENTITY ); } }); it('mints nothing a world cannot read', () => { - expect( - requiresNewerWorld(mintedSpecVersion({ [SLOT_IDENTITY_ENV_VAR]: '1' })) - ).toBe(false); + expect(requiresNewerWorld(mintedSpecVersion({}))).toBe(false); }); }); diff --git a/packages/world/src/spec-version.ts b/packages/world/src/spec-version.ts index a86f4f3e90..4b830fd4a8 100644 --- a/packages/world/src/spec-version.ts +++ b/packages/world/src/spec-version.ts @@ -49,10 +49,10 @@ export const SPEC_VERSION_SLOT_IDENTITY = 6 as SpecVersion; * Current spec version (event-sourced architecture with native attributes * and compressed payloads). * - * This is the version new runs are stamped with, which is a *lower* bar than - * the newest version this build can read — see - * {@link SPEC_VERSION_MAX_SUPPORTED}. Slot identity ships behind a flag, so it - * is readable everywhere before it is minted anywhere. + * The floor a world stamps on new runs, and a *lower* bar than the newest + * version this build can read — see {@link SPEC_VERSION_MAX_SUPPORTED}. What a + * world actually stamps comes from {@link mintedSpecVersion}; this is what it + * falls back to when slot identity is switched off. */ export const SPEC_VERSION_CURRENT = SPEC_VERSION_SUPPORTS_COMPRESSION as SpecVersion; @@ -62,16 +62,15 @@ export const SPEC_VERSION_CURRENT = * by {@link requiresNewerWorld} rather than misread. * * Distinct from {@link SPEC_VERSION_CURRENT} because a world has to be able to - * read a version before anything may mint it: the flag that turns slot identity - * on for new runs would otherwise make every world reject the runs it had just - * created. Worlds opt into minting individually, via the `specVersion` they - * declare. + * read a version before anything may mint it, and because a world that mints + * slot identity still has to read the spec-5 runs it created before the switch. + * Worlds opt into minting individually, via the `specVersion` they declare. */ export const SPEC_VERSION_MAX_SUPPORTED = SPEC_VERSION_SLOT_IDENTITY as SpecVersion; /** - * Environment variable that opts new runs into slot identity. + * Environment variable that opts new runs out of slot identity. * * Read per `createWorld()` call rather than at module load, so a test or a * single process can create worlds in both modes. @@ -80,20 +79,20 @@ export const SLOT_IDENTITY_ENV_VAR = 'WORKFLOW_SLOT_IDENTITY'; /** * The spec version a world should stamp on the runs it creates: slot identity - * when {@link SLOT_IDENTITY_ENV_VAR} is set, otherwise + * unless {@link SLOT_IDENTITY_ENV_VAR} disables it, in which case * {@link SPEC_VERSION_CURRENT}. * * Every world reads runs up to {@link SPEC_VERSION_MAX_SUPPORTED} whatever this - * returns, so turning the flag on in one place does not make the runs it creates - * unreadable elsewhere. + * returns, so turning the flag off in one place does not make the runs another + * process created unreadable here. */ export function mintedSpecVersion( env: Record = process.env ): SpecVersion { const value = env[SLOT_IDENTITY_ENV_VAR]; - return value === '1' || value === 'true' - ? SPEC_VERSION_SLOT_IDENTITY - : SPEC_VERSION_CURRENT; + return value === '0' || value === 'false' + ? SPEC_VERSION_CURRENT + : SPEC_VERSION_SLOT_IDENTITY; } /** From df22f35ebbf302373bb67b16440a98f9ea0fee0c Mon Sep 17 00:00:00 2001 From: Peter Wielander Date: Thu, 30 Jul 2026 23:20:10 -0700 Subject: [PATCH 11/35] fix(world-local): re-probe a lost position instead of conflicting MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit A writer with no event log — a step completion reporting in, a hook being received, a cancellation from an API call — has nothing to reconcile when the position it allocated turns out to be taken, so surfacing a slot conflict to it strands the caller with an error it cannot act on. Two world instances sharing a data directory keep independent books, which makes that the common case rather than a rarity. The publish is now one attempt in a loop: an allocated position that loses tops the book up from disk, backs off with full jitter, and takes the next free one, within a 30s budget after which the write fails retryably. A position the caller claimed still conflicts on the first loss — the claim asserts a complete log, so only the caller can resolve it. The contention profile moves to @workflow/world so the Local and Postgres Worlds share one definition. --- .../world-local/src/storage/events-storage.ts | 301 +++++++++++------- .../src/storage/slot-identity.test.ts | 39 ++- packages/world-local/src/storage/slots.ts | 28 ++ packages/world-postgres/src/slots.ts | 29 +- packages/world/src/index.ts | 4 + packages/world/src/slot-identity.ts | 25 ++ 6 files changed, 273 insertions(+), 153 deletions(-) diff --git a/packages/world-local/src/storage/events-storage.ts b/packages/world-local/src/storage/events-storage.ts index 98e705e98d..72c8757a46 100644 --- a/packages/world-local/src/storage/events-storage.ts +++ b/packages/world-local/src/storage/events-storage.ts @@ -37,9 +37,11 @@ import { requiresNewerWorld, SPEC_VERSION_CURRENT, SPEC_VERSION_MAX_SUPPORTED, + SLOT_RETRY_BUDGET_MS, StepSchema, slotEventId, slotFromId, + slotRetryDelay, ulidToDate, usesSlotIdentity, validateAttributeChanges, @@ -1009,6 +1011,13 @@ export function createEventsStorage( // ============================================================ // EVENT ID: the caller's slot claim, an allocated slot, or a ULID // ============================================================ + // A run's own `run_created` owns the first slot — provably, since + // nothing precedes it — so every other event allocates above it, even + // when that event is the first to arrive here. + const allocationFloor = + data.eventType === 'run_created' + ? RUN_CREATED_SLOT + : RUN_CREATED_SLOT + 1; // A slot-numbered run's ids name positions in its log, so an id is // either claimed by a caller that holds the log (and is therefore // asserting the log is complete up to that position) or allocated here @@ -1042,15 +1051,7 @@ export function createEventsStorage( } } else if (slotMode) { reservedRunId = effectiveRunId; - // A run's own `run_created` owns the first slot — provably, since - // nothing precedes it — so every other event allocates above it, even - // when that event is the first to arrive here. - const slot = await slots.reserve( - effectiveRunId, - data.eventType === 'run_created' - ? RUN_CREATED_SLOT - : RUN_CREATED_SLOT + 1 - ); + const slot = await slots.reserve(effectiveRunId, allocationFloor); reserved.add(slot); eventId = slotEventId(slot); } @@ -2457,120 +2458,172 @@ export function createEventsStorage( throw new HookNotFoundError(data.correlationId); } - const compositeKey = `${effectiveRunId}-${eventId}`; - const eventPath = taggedPath(basedir, 'events', compositeKey, tag); - // Capture the serialized payload before the write's `await` so the - // cached snapshot can't observe a later mutation (see - // rememberStoredEvent). - const serializedEvent = JSON.stringify(event, jsonReplacer, 2); - - // Cross-process terminal-run guard for `hook_received`. A terminal - // transition (run_completed / run_failed / run_cancelled) in ANY - // process (1) publishes a durable `runTerminalMarkerPath` marker and - // (2) reaps the run's staged hook_received events, both BEFORE it - // writes the terminal run state or appends its terminal event (see - // the terminal-transition block earlier in this function). In-memory - // locks cannot close the shared-filesystem race this backend - // explicitly supports, and a published event file is immediately - // visible to `events.list()` in other processes — so it can never be - // "rolled back" after the fact. Instead, the event stays INVISIBLE - // to readers until a single atomic filesystem operation decides its - // fate: - // - // 1. (fast path) reject if the run is already terminal — by - // marker, or by run state for runs that predate the marker — - // so the common case never creates a file. - // 2. STAGE the event at a non-reader-visible path under `.locks`. - // 3. re-CHECK the terminal marker; reject if present. - // 4. PROMOTE the staged file into `events/` with an atomic hard - // link; reject if the staged file was reaped (`'missing'`). - // - // Correctness: the reap's `unlink` and step 4's `link` target the - // same staged file, so the filesystem serializes them — exactly one - // wins. If the link wins, the event was reader-visible before the - // reap completed, and therefore before the terminal state and - // terminal event were written: acceptance happened-before the - // termination and legitimately precedes it. If the unlink wins, - // promotion fails and the event is never visible to any reader — - // there is nothing to roll back. A resume that stages after the - // reap has passed necessarily stages after the marker was - // committed, so step 3 rejects it. Rejections before step 4 unlink - // a file no reader can see. - let eventPublished: boolean; - if (data.eventType === 'hook_received') { - // Step 1: fast path. The marker is the authoritative durable - // signal; the run-state read additionally rejects runs whose - // terminal state was written without a marker (e.g. runs that - // terminated on an older storage version). - const terminalByMarker = await isRunTerminalCommitted( - basedir, - effectiveRunId, - tag - ); - const runNow = terminalByMarker - ? null - : await readJSONWithFallback( - basedir, - 'runs', - effectiveRunId, - WorkflowRunSchema, - tag - ); - if ( - terminalByMarker || - (runNow && isTerminalWorkflowRunStatus(runNow.status)) - ) { - throw new RunExpiredError( - `Workflow run "${effectiveRunId}" is already in a terminal state` - ); - } - - const stagedPath = pendingHookEventPath( - basedir, - effectiveRunId, - eventId, - tag - ); - const staged = await writeExclusive(stagedPath, serializedEvent); - if (!staged) { - // For a ULID-numbered run the eventId is freshly generated, so - // its staging path can only be occupied by a previous crashed - // attempt of this very event, which never promoted. A - // slot-numbered run can also collide here with another instance - // that allocated the same slot from its own book. Either way the - // event is not reader-visible, so there is no delta to hand back - // and nothing for the caller to merge: surface the same conflict - // shape as a visible-path collision, and let the allocator - // re-probe on the retry. - throw new EntityConflictError( - `Event "${eventId}" already exists for run "${effectiveRunId}"` + /** + * One attempt at publishing the event at the position `eventId` + * currently names: `true` when this call made it reader-visible, + * `false` when the position was already taken. What a loss means is the + * loop's decision — a position this world allocated is simply retried + * one higher, a position the caller claimed is a conflict it has to + * resolve. + */ + async function publishOnce(): Promise { + // Cross-process terminal-run guard for `hook_received`. A terminal + // transition (run_completed / run_failed / run_cancelled) in ANY + // process (1) publishes a durable `runTerminalMarkerPath` marker and + // (2) reaps the run's staged hook_received events, both BEFORE it + // writes the terminal run state or appends its terminal event (see + // the terminal-transition block earlier in this function). In-memory + // locks cannot close the shared-filesystem race this backend + // explicitly supports, and a published event file is immediately + // visible to `events.list()` in other processes — so it can never be + // "rolled back" after the fact. Instead, the event stays INVISIBLE + // to readers until a single atomic filesystem operation decides its + // fate: + // + // 1. (fast path) reject if the run is already terminal — by + // marker, or by run state for runs that predate the marker — + // so the common case never creates a file. + // 2. STAGE the event at a non-reader-visible path under `.locks`. + // 3. re-CHECK the terminal marker; reject if present. + // 4. PROMOTE the staged file into `events/` with an atomic hard + // link; reject if the staged file was reaped (`'missing'`). + // + // Correctness: the reap's `unlink` and step 4's `link` target the + // same staged file, so the filesystem serializes them — exactly one + // wins. If the link wins, the event was reader-visible before the + // reap completed, and therefore before the terminal state and + // terminal event were written: acceptance happened-before the + // termination and legitimately precedes it. If the unlink wins, + // promotion fails and the event is never visible to any reader — + // there is nothing to roll back. A resume that stages after the + // reap has passed necessarily stages after the marker was + // committed, so step 3 rejects it. Rejections before step 4 unlink + // a file no reader can see. + if (data.eventType === 'hook_received') { + // Step 1: fast path. The marker is the authoritative durable + // signal; the run-state read additionally rejects runs whose + // terminal state was written without a marker (e.g. runs that + // terminated on an older storage version). + const terminalByMarker = await isRunTerminalCommitted( + basedir, + effectiveRunId, + tag ); - } - try { - if (await isRunTerminalCommitted(basedir, effectiveRunId, tag)) { + const runNow = terminalByMarker + ? null + : await readJSONWithFallback( + basedir, + 'runs', + effectiveRunId, + WorkflowRunSchema, + tag + ); + if ( + terminalByMarker || + (runNow && isTerminalWorkflowRunStatus(runNow.status)) + ) { throw new RunExpiredError( `Workflow run "${effectiveRunId}" is already in a terminal state` ); } - const promoted = await promoteExclusive(stagedPath, eventPath); - if (promoted === 'missing') { - // A terminal transition reaped the staged file between the - // check and the link — the atomic loss of the arbitration. - throw new RunExpiredError( - `Workflow run "${effectiveRunId}" is already in a terminal state` + + const stagedPath = pendingHookEventPath( + basedir, + effectiveRunId, + eventId, + tag + ); + const staged = await writeExclusive(stagedPath, serializedEvent); + if (!staged) { + // For a ULID-numbered run the eventId is freshly generated, so + // its staging path can only be occupied by a previous crashed + // attempt of this very event, which never promoted. A + // slot-numbered run can also collide here with another instance + // that allocated the same slot from its own book. Either way the + // event is not reader-visible, so there is no delta to hand back + // and nothing for the caller to merge: surface the same conflict + // shape as a visible-path collision — or, when this world + // allocated the position itself, let the loop below re-probe and + // take the next one. + if (reallocatesSlot) { + return false; + } + throw new EntityConflictError( + `Event "${eventId}" already exists for run "${effectiveRunId}"` ); } - eventPublished = promoted === 'linked'; - } finally { - // The staged path is not reader-visible; removing it is pure - // cleanup on every outcome (already gone when reaped). - await deleteJSON(stagedPath).catch(() => {}); + try { + if (await isRunTerminalCommitted(basedir, effectiveRunId, tag)) { + throw new RunExpiredError( + `Workflow run "${effectiveRunId}" is already in a terminal state` + ); + } + const promoted = await promoteExclusive(stagedPath, eventPath); + if (promoted === 'missing') { + // A terminal transition reaped the staged file between the + // check and the link — the atomic loss of the arbitration. + throw new RunExpiredError( + `Workflow run "${effectiveRunId}" is already in a terminal state` + ); + } + return promoted === 'linked'; + } finally { + // The staged path is not reader-visible; removing it is pure + // cleanup on every outcome (already gone when reaped). + await deleteJSON(stagedPath).catch(() => {}); + } } - } else { - eventPublished = await writeExclusive(eventPath, serializedEvent); + return await writeExclusive(eventPath, serializedEvent); } - if (!eventPublished) { + // A write that allocated its own position may take the next free one + // when it loses: nothing outside this world named the slot, so which + // position the event lands on is this world's business, and the caller + // — a step reporting its completion, a hook being received — has no log + // to reconcile. A write whose position the *caller* claimed may not: + // the claim asserts a log complete up to that position, so losing it + // means that log is stale and only the caller can resolve it. + const reallocatesSlot = slotMode && params?.eventId === undefined; + const slotDeadline = Date.now() + SLOT_RETRY_BUDGET_MS; + let compositeKey = ''; + let eventPath = ''; + let serializedEvent = ''; + let eventPublished = false; + + for (let round = 0; ; round++) { + compositeKey = `${effectiveRunId}-${eventId}`; + eventPath = taggedPath(basedir, 'events', compositeKey, tag); + // Capture the serialized payload before the write's `await` so the + // cached snapshot can't observe a later mutation (see + // rememberStoredEvent). + serializedEvent = JSON.stringify(event, jsonReplacer, 2); + eventPublished = await publishOnce(); + if (eventPublished) { + break; + } + if (reallocatesSlot && Date.now() < slotDeadline) { + // The position is someone else's — either published there or + // staged for it. Record that, top the book up from disk, and try + // again one position higher rather than surfacing a conflict the + // caller cannot act on. Re-reading rather than incrementing keeps + // the log dense: every round at least one writer wins, so the + // search never runs away from the log it is filling. + const lost = slotFromId(eventId); + if (lost !== undefined) { + reserved.delete(lost); + } + slots.observe(effectiveRunId, eventId); + await slots.refresh(effectiveRunId); + await new Promise((resolve) => + setTimeout(resolve, slotRetryDelay(round)) + ); + const slot = await slots.reserve(effectiveRunId, allocationFloor); + reservedRunId = effectiveRunId; + reserved.add(slot); + eventId = slotEventId(slot); + event = { ...event, eventId }; + continue; + } // For `hook_created`, losing the event publish means the // event was already committed at this exact (canonical) // path. The original publisher may have crashed between @@ -2591,13 +2644,23 @@ export function createEventsStorage( tag ); } + if (reallocatesSlot) { + // Out of budget: every position this writer tried was taken by + // someone else. Surfacing it as a 503 puts the whole operation + // back on the queue rather than stalling the run here. + throw new WorkflowWorldError( + `Could not place an event in run "${effectiveRunId}" within ${SLOT_RETRY_BUDGET_MS}ms of contention`, + { status: 503 } + ); + } if (slotMode) { - // Losing a slot means someone else's event occupies this position, - // so the log this event was derived from is missing at least that - // event — the whole proposed event is stale, not just its id. Hand - // back what the caller is missing so it can merge, replay and - // re-propose, and forget the run's book so the next allocation - // re-reads the log this instance evidently does not have. + // Losing a claimed slot means someone else's event occupies this + // position, so the log this event was derived from is missing at + // least that event — the whole proposed event is stale, not just + // its id. Hand back what the caller is missing so it can merge, + // replay and re-propose, and forget the run's book so the next + // allocation re-reads the log this instance evidently does not + // have. // // Reaching here means the slot was taken *after* the pre-check at // the claim site, so the entity this event was going to describe diff --git a/packages/world-local/src/storage/slot-identity.test.ts b/packages/world-local/src/storage/slot-identity.test.ts index 00d8b71cce..f936b55718 100644 --- a/packages/world-local/src/storage/slot-identity.test.ts +++ b/packages/world-local/src/storage/slot-identity.test.ts @@ -249,12 +249,31 @@ describe('conflict', () => { ).toEqual([slotEventId(3)]); }); - it('conflicts across two storage instances sharing a directory', async () => { + it('conflicts when another instance takes a claimed slot', async () => { // Two instances keep independent books, so the exclusive write — not the - // book — is what decides who owns a slot. + // book — is what decides who owns a slot. A claim asserts a complete log, + // so its loser has to reload rather than move over. const runId = await newSlotRun(); const other = createStorage(testDir); - const [first, second] = await Promise.allSettled([ + await other.events.create(runId, { + eventType: 'step_created', + specVersion: SPEC_VERSION_SLOT_IDENTITY, + correlationId: 'step_b', + eventData: { stepName: 'b-step', input: new Uint8Array() }, + }); + await expect(createStep(runId, 'step_a', slotEventId(2))).rejects.toThrow( + SlotConflictError + ); + await expect(slotsOf(runId)).resolves.toEqual([1, 2]); + }); + + it('reallocates around another instance holding the slot it picked', async () => { + // Neither writer holds a log, so neither has anything to reconcile: the + // loser takes the next free position instead of surfacing a conflict its + // caller could not act on. + const runId = await newSlotRun(); + const other = createStorage(testDir); + const outcomes = await Promise.allSettled([ storage.events.create(runId, { eventType: 'step_created', specVersion: SPEC_VERSION_SLOT_IDENTITY, @@ -268,14 +287,10 @@ describe('conflict', () => { eventData: { stepName: 'b-step', input: new Uint8Array() }, }), ]); - const outcomes = [first, second]; - expect(outcomes.filter((o) => o.status === 'fulfilled')).toHaveLength(1); - const rejection = outcomes.find((o) => o.status === 'rejected'); - expect( - SlotConflictError.is( - (rejection as PromiseRejectedResult | undefined)?.reason - ) - ).toBe(true); - await expect(slotsOf(runId)).resolves.toEqual([1, 2]); + expect(outcomes.map((outcome) => outcome.status)).toEqual([ + 'fulfilled', + 'fulfilled', + ]); + await expect(slotsOf(runId)).resolves.toEqual([1, 2, 3]); }); }); diff --git a/packages/world-local/src/storage/slots.ts b/packages/world-local/src/storage/slots.ts index c6b2f017f6..632e31c1ee 100644 --- a/packages/world-local/src/storage/slots.ts +++ b/packages/world-local/src/storage/slots.ts @@ -96,6 +96,16 @@ export interface SlotBook { release(runId: string, slot: number): void; /** Records a published event id, so it is never handed out again. */ observe(runId: string, eventId: string): void; + /** + * Merges the run's published positions from disk into the book kept for it, + * leaving the reservations other writers in this instance still hold. + * + * A writer whose publish lost its position calls this before trying again: + * the book is demonstrably behind another instance's writes, and dropping it + * wholesale ({@link SlotBook.forget}) would hand a sibling's outstanding + * position to the next caller and cost that sibling its own publish. + */ + refresh(runId: string): Promise; /** Drops what is cached for `runId`, so the next reservation re-reads disk. */ forget(runId: string): void; /** Drops everything cached (the data directory was cleared out from under us). */ @@ -227,6 +237,24 @@ export function createSlotBook(basedir: string, tag?: string): SlotBook { book.outstanding.delete(slot); }, + async refresh(runId) { + const book = books.get(runId); + if (!book) { + // Nothing cached to correct; the next reservation seeds from disk. + return; + } + for (const eventId of await listRunEventIds(basedir, runId, tag)) { + const slot = slotFromId(eventId); + if (slot !== undefined) { + book.written.add(slot); + book.outstanding.delete(slot); + } + } + // Positions released while this scan ran may sit below where the search + // had reached, and they are free again. + book.searchFrom = FIRST_SLOT; + }, + forget(runId) { modes.delete(runId); books.delete(runId); diff --git a/packages/world-postgres/src/slots.ts b/packages/world-postgres/src/slots.ts index 89435810a9..2f30ad9f31 100644 --- a/packages/world-postgres/src/slots.ts +++ b/packages/world-postgres/src/slots.ts @@ -14,7 +14,13 @@ */ import { WorkflowWorldError } from '@workflow/errors'; -import { FIRST_SLOT, slotEventId, slotFromId } from '@workflow/world'; +import { + FIRST_SLOT, + SLOT_RETRY_BUDGET_MS, + slotEventId, + slotFromId, + slotRetryDelay, +} from '@workflow/world'; import { and, desc, eq } from 'drizzle-orm'; import { type Drizzle, Schema } from './drizzle/index.js'; @@ -26,19 +32,6 @@ import { type Drizzle, Schema } from './drizzle/index.js'; */ export const RUN_CREATED_SLOT = FIRST_SLOT; -/** First backoff after losing a position; doubled each round. */ -export const SLOT_RETRY_BASE_MS = 5; - -/** Ceiling for a single backoff, so a contended run keeps making attempts. */ -export const SLOT_RETRY_MAX_DELAY_MS = 250; - -/** - * How long a writer keeps looking for a free position before giving up. - * Exhausting it surfaces as a 503, so the caller — in practice a queue - * delivery — retries the whole operation instead of the run stalling on it. - */ -export const SLOT_RETRY_BUDGET_MS = 30_000; - /** Postgres unique-violation code. */ const UNIQUE_VIOLATION = '23505'; @@ -104,14 +97,6 @@ export async function eventExists( return row !== undefined; } -/** Full jitter over an exponentially growing, capped window. */ -export function slotRetryDelay(round: number): number { - return ( - Math.random() * - Math.min(SLOT_RETRY_BASE_MS * 2 ** round, SLOT_RETRY_MAX_DELAY_MS) - ); -} - /** The event ids a single create publishes. */ export interface EventIds { /** diff --git a/packages/world/src/index.ts b/packages/world/src/index.ts index 5a8c185f2a..1ecae0457d 100644 --- a/packages/world/src/index.ts +++ b/packages/world/src/index.ts @@ -106,9 +106,13 @@ export { isSlotId, maxSlotOf, SLOT_ID_WIDTH, + SLOT_RETRY_BASE_MS, + SLOT_RETRY_BUDGET_MS, + SLOT_RETRY_MAX_DELAY_MS, slotEventId, slotFromId, slotIdBody, + slotRetryDelay, } from './slot-identity.js'; export type { SpecVersion } from './spec-version.js'; export { diff --git a/packages/world/src/slot-identity.ts b/packages/world/src/slot-identity.ts index d9ff11b65f..d761f91452 100644 --- a/packages/world/src/slot-identity.ts +++ b/packages/world/src/slot-identity.ts @@ -67,6 +67,31 @@ export function slotEventId(slot: number): string { return `evnt_${slotIdBody(slot)}`; } +/** First backoff after losing a position; doubled each round. */ +export const SLOT_RETRY_BASE_MS = 5; + +/** Ceiling for a single backoff, so a contended run keeps making attempts. */ +export const SLOT_RETRY_MAX_DELAY_MS = 250; + +/** + * How long a writer that allocates its own position keeps looking for a free + * one before giving up. Exhausting it is a retryable failure for the caller — + * in practice a queue delivery — rather than something the run stalls on. + */ +export const SLOT_RETRY_BUDGET_MS = 30_000; + +/** + * Full jitter over an exponentially growing, capped window. Shared by every + * world that allocates positions, so contention behaves the same wherever a run + * is stored. + */ +export function slotRetryDelay(round: number): number { + return ( + Math.random() * + Math.min(SLOT_RETRY_BASE_MS * 2 ** round, SLOT_RETRY_MAX_DELAY_MS) + ); +} + /** * The highest slot named by any of `events`, or 0 when none is slot-numbered. * From 531b15ec082fb269f477a5ff1fdad0f68aacd05d Mon Sep 17 00:00:00 2001 From: Peter Wielander Date: Fri, 31 Jul 2026 00:37:37 -0700 Subject: [PATCH 12/35] fix(worlds): number the deferred step_created below the claim it rides with A lazy start publishes two events, and only the caller knows which positions are free: it hands out a whole batch's slots synchronously, before any of them land. So a claim names the top of the pair and the deferred `step_created` takes the position immediately below it, which the caller reserved for exactly that. A start that claimed nothing has both positions allocated here instead, and a claim with no room below it is rejected rather than allowed to overwrite the run's creation. Co-Authored-By: Claude Opus 5 --- .../world-local/src/storage/events-storage.ts | 82 +++++++++++++++---- .../src/storage/slot-identity.test.ts | 78 ++++++++++++++++++ packages/world-postgres/src/slots.ts | 44 +++++++--- packages/world-postgres/src/storage.ts | 9 +- .../world-postgres/test/slot-identity.test.ts | 63 ++++++++++++-- 5 files changed, 235 insertions(+), 41 deletions(-) diff --git a/packages/world-local/src/storage/events-storage.ts b/packages/world-local/src/storage/events-storage.ts index 72c8757a46..558d5f7d03 100644 --- a/packages/world-local/src/storage/events-storage.ts +++ b/packages/world-local/src/storage/events-storage.ts @@ -1008,6 +1008,17 @@ export function createEventsStorage( } } + // Lazy step start: a step_started carrying step-creation data + // (stepName + input) is allowed to arrive with no prior step_created + // — it creates the step on the fly (see the materialization block + // below). This mirrors the resilient run_started path. Detect it here + // so the second event it publishes can be numbered alongside the + // first, the entity-creation terminal-run guard treats it like a + // creation, and the "step must exist" ordering guard doesn't reject it. + const createsChildEntity = isChildEntityCreationEvent(data); + const lazyStepStart = + createsChildEntity && data.eventType === 'step_started'; + // ============================================================ // EVENT ID: the caller's slot claim, an allocated slot, or a ULID // ============================================================ @@ -1023,6 +1034,11 @@ export function createEventsStorage( // asserting the log is complete up to that position) or allocated here // for a caller that has no log — a step completion reporting in, a // cancellation from an API call. + // + // The position of the second event a lazy start publishes, when this + // one publishes two. Consumed by the materialization below; released + // again if that block turns out not to need it. + let companionSlot: number | undefined; if (params?.eventId !== undefined) { const claimedSlot = slotFromId(params.eventId); if (!slotMode) { @@ -1041,13 +1057,38 @@ export function createEventsStorage( reservedRunId = effectiveRunId; reserved.add(claimedSlot); slots.claim(effectiveRunId, claimedSlot); - if (await slots.isWritten(effectiveRunId, claimedSlot)) { - // Reject a doomed claim before the materialization below creates - // the step, hook or wait this event will now never accompany. A - // caller that re-proposes at the next slot would otherwise - // collide with its own orphan and read that as "my write already - // landed". See SlotBook.isWritten. - throw await slotConflict(effectiveRunId, eventId, params); + // One request, two events: a lazy start also publishes the + // `step_created` it deferred. A claim names the *top* of the pair, + // so the second event takes the slot immediately below it — the + // caller reserved both positions and named only one, which is what + // keeps the pair from landing on a position another write in the + // same batch is already holding. + if (lazyStepStart) { + companionSlot = claimedSlot - 1; + if (companionSlot < RUN_CREATED_SLOT + 1) { + throw new WorkflowWorldError( + `Event id "${params.eventId}" leaves no slot below it in run "${effectiveRunId}" for the "step_created" published alongside it`, + { status: 400 } + ); + } + reserved.add(companionSlot); + slots.claim(effectiveRunId, companionSlot); + } + // Reject a doomed claim before the materialization below creates + // the step, hook or wait this event will now never accompany. A + // caller that re-proposes at the next slot would otherwise + // collide with its own orphan and read that as "my write already + // landed". See SlotBook.isWritten. + for (const slot of companionSlot === undefined + ? [claimedSlot] + : [companionSlot, claimedSlot]) { + if (await slots.isWritten(effectiveRunId, slot)) { + throw await slotConflict( + effectiveRunId, + slotEventId(slot), + params + ); + } } } else if (slotMode) { reservedRunId = effectiveRunId; @@ -1060,16 +1101,6 @@ export function createEventsStorage( // VALIDATION: Terminal state and event ordering checks // ============================================================ - // Lazy step start: a step_started carrying step-creation data - // (stepName + input) is allowed to arrive with no prior step_created - // — it creates the step on the fly (see the materialization block - // below). This mirrors the resilient run_started path. Detect it here - // so the entity-creation terminal-run guard treats it like a creation - // and the "step must exist" ordering guard doesn't reject it. - const createsChildEntity = isChildEntityCreationEvent(data); - const lazyStepStart = - createsChildEntity && data.eventType === 'step_started'; - // Run terminal state validation if (currentRun && isTerminalWorkflowRunStatus(currentRun.status)) { // Idempotent operation: run_cancelled on already cancelled run is allowed @@ -1724,7 +1755,13 @@ export function createEventsStorage( // run_started → run_created precedent in this file. let stepCreatedEventId = `evnt_${monotonicUlid()}`; if (slotMode) { - const slot = await slots.reserve(effectiveRunId); + // A claimed start numbers this event one below its own + // position, which the caller reserved for exactly this. A start + // that allocated takes the next free slot instead: nothing + // outside this world named either position. + const slot = + companionSlot ?? (await slots.reserve(effectiveRunId)); + companionSlot = undefined; reserved.add(slot); stepCreatedEventId = slotEventId(slot); } @@ -2679,6 +2716,15 @@ export function createEventsStorage( // replay can serve it without rereading from disk. rememberStoredEvent(event, eventPath, serializedEvent); slots.observe(effectiveRunId, eventId); + if (companionSlot !== undefined) { + // A start that carried creation data for a step that already existed + // synthesized no `step_created`, so the position below it went + // unused. Hand it back instead of leaving it outstanding for the life + // of the process, where it would block the allocator from ever + // filling that position. + reserved.delete(companionSlot); + slots.release(effectiveRunId, companionSlot); + } // Write the hook entity ONLY now that the event publish has // committed. Doing this earlier (in the `hook_created` diff --git a/packages/world-local/src/storage/slot-identity.test.ts b/packages/world-local/src/storage/slot-identity.test.ts index f936b55718..885126e5fa 100644 --- a/packages/world-local/src/storage/slot-identity.test.ts +++ b/packages/world-local/src/storage/slot-identity.test.ts @@ -147,6 +147,84 @@ describe('numbering', () => { }); }); +/** + * A lazy step start: a `step_started` carrying the step's creation data, which + * the world materializes into a step plus the `step_created` event the caller + * deferred — one request, two events. + */ +async function startStepLazily( + runId: string, + stepId: string, + eventId?: string +): Promise { + const result = await storage.events.create( + runId, + { + eventType: 'step_started', + specVersion: SPEC_VERSION_SLOT_IDENTITY, + correlationId: stepId, + eventData: { stepName: 'a-step', input: new Uint8Array(), attempt: 0 }, + }, + eventId === undefined ? undefined : { eventId } + ); + if (!result.event) { + throw new Error('Expected an event'); + } + return result.event.eventId; +} + +describe('a write that publishes two events', () => { + it('numbers the deferred step_created below the claim', async () => { + // The caller reserves both positions and names only the top one, so the + // pair is fixed before either lands — which is what keeps it off the slot + // the next write of the same batch is holding. + const runId = await newSlotRun(); + const startedEventId = await startStepLazily( + runId, + 'step_a', + slotEventId(3) + ); + expect(startedEventId).toBe(slotEventId(3)); + await expect(slotsOf(runId)).resolves.toEqual([1, 2, 3]); + }); + + it('allocates both positions for a start that claims neither', async () => { + const runId = await newSlotRun(); + await startStepLazily(runId, 'step_a'); + await expect(slotsOf(runId)).resolves.toEqual([1, 2, 3]); + }); + + it('keeps every claim in a burst of lazy starts', async () => { + // The suspension flush issues its lazy starts at once, each having reserved + // two positions. A second event numbered off the log as this world sees it + // would take the slot the next start in the batch claimed, and cost every + // start after the first its claim — collapsing the fan-out to one step. + const runId = await newSlotRun(); + const claims = Array.from({ length: 10 }, (_, index) => + slotEventId(FIRST_SLOT + 2 * (index + 1)) + ); + const ids = await Promise.all( + claims.map((eventId, index) => + startStepLazily(runId, `step_${index}`, eventId) + ) + ); + expect(ids).toEqual(claims); + const slots = await slotsOf(runId); + expect([...slots].sort((a, b) => a - b)).toEqual( + Array.from({ length: 2 * claims.length + 1 }, (_, i) => FIRST_SLOT + i) + ); + }); + + it('rejects a claim that leaves no room for the second event', async () => { + // The run's own run_created holds the first slot, so a claim of the second + // means the caller reserved one position for a write that publishes two. + const runId = await newSlotRun(); + await expect( + startStepLazily(runId, 'step_a', slotEventId(FIRST_SLOT + 1)) + ).rejects.toThrow(/leaves no slot below it/); + }); +}); + describe('mode is pinned to the run', () => { it('rejects a slot id claimed on a ULID-numbered run', async () => { const created = await storage.events.create(null, { diff --git a/packages/world-postgres/src/slots.ts b/packages/world-postgres/src/slots.ts index 2f30ad9f31..cd0a54f6f5 100644 --- a/packages/world-postgres/src/slots.ts +++ b/packages/world-postgres/src/slots.ts @@ -107,9 +107,12 @@ export interface EventIds { primary: () => string; /** * An additional event written in the same breath — the synthetic - * `step_created` of a lazy step start. Its position is allocated here even - * when the caller named its own for the primary event, because a caller that - * defers a `step_created` cannot know it will be synthesized. + * `step_created` of a lazy step start. + * + * A claim names the *top* of the pair, so this event takes the position + * immediately below it: the caller reserved both and named one. Numbering it + * off the log instead would hand it a position another write of the same + * concurrent batch is already holding, and cost that write its claim. */ extra: () => Promise; } @@ -178,23 +181,40 @@ export async function placeEvent( options.claimedSlot === undefined ? slotEventId(await take()) : slotEventId(options.claimedSlot); + /** Positions the caller named, which are the caller's to resolve. */ + const claimed = options.claimedSlot === undefined ? [] : [primary]; try { return await options.write({ primary: () => primary, - extra: async () => slotEventId(await take()), + extra: async () => { + if (options.claimedSlot === undefined) { + return slotEventId(await take()); + } + const slot = options.claimedSlot - 1; + if (slot <= FIRST_SLOT) { + // The run's own `run_created` holds the first slot, so a claim of + // the second leaves nowhere for a second event to go: the caller + // reserved one position for a write that publishes two. + throw new WorkflowWorldError( + `Event id "${primary}" leaves no slot below it in run "${runId}" for the second event published alongside it`, + { status: 400 } + ); + } + const id = slotEventId(slot); + claimed.push(id); + return id; + }, }); } catch (error) { if (!isEventKeyViolation(error)) { throw error; } - // A claimed write can also lose on its extra event's position, which is - // this world's to reallocate — only a claim that is itself taken is the - // caller's problem. - if ( - options.claimedSlot !== undefined && - (await eventExists(drizzle, runId, primary)) - ) { - throw await options.onClaimTaken(); + // Only a position the caller named is the caller's problem; one this + // world allocated is reallocated below without ever surfacing. + for (const id of claimed) { + if (await eventExists(drizzle, runId, id)) { + throw await options.onClaimTaken(); + } } if (Date.now() >= deadline) { throw new WorkflowWorldError( diff --git a/packages/world-postgres/src/storage.ts b/packages/world-postgres/src/storage.ts index 878f39758b..ab752f7de5 100644 --- a/packages/world-postgres/src/storage.ts +++ b/packages/world-postgres/src/storage.ts @@ -1411,11 +1411,10 @@ export function createEventsStorage(drizzle: Drizzle): Storage['events'] { // side of the materialization would, hence the shared // transaction. // - // It takes a position of its own, which on a claimed write is - // necessarily one this world allocated: a caller that defers a - // step_created cannot know it will be synthesized, so it never - // claims a slot for it. Losing that position rolls the transaction - // back for a retry, so the insert must not swallow the collision. + // It takes a position of its own — the one below the claim on a + // claimed write, the next free one otherwise. Losing that position + // rolls the transaction back for a retry, so the insert must not + // swallow the collision. const stepCreatedEventId = await ids.extra(); const insertStepCreated = tx.insert(events).values({ runId: effectiveRunId, diff --git a/packages/world-postgres/test/slot-identity.test.ts b/packages/world-postgres/test/slot-identity.test.ts index d8b0578a7d..8e3f04c8a4 100644 --- a/packages/world-postgres/test/slot-identity.test.ts +++ b/packages/world-postgres/test/slot-identity.test.ts @@ -173,9 +173,10 @@ describe('Slot identity (Postgres integration)', () => { ).toEqual(['1 run_created', '2 step_started', '3 step_created']); }); - test('allocates the deferred step_created above a claimed slot', async () => { - // A caller that defers a step_created cannot know it will be synthesized, - // so it never claims a slot for it — and the slot it did claim is its own. + test('numbers the deferred step_created below a claimed slot', async () => { + // A claim names the top of the pair: the caller reserved both positions + // before either landed, which is what keeps the second event off the slot + // the next write of the same batch claimed. const runId = await newSlotRun(); const started = await events.create( runId, @@ -185,10 +186,60 @@ describe('Slot identity (Postgres integration)', () => { correlationId: 'step_a', eventData: { stepName: 'a-step', input: new Uint8Array() }, }, - { eventId: slotEventId(2) } + { eventId: slotEventId(3) } ); - expect(started.event?.eventId).toBe(slotEventId(2)); - expect(ascending(await slotsOf(runId))).toEqual(denseFrom(3)); + expect(started.event?.eventId).toBe(slotEventId(3)); + const { data } = await eventsOf(runId); + expect( + data.map((event) => `${slotFromId(event.eventId)} ${event.eventType}`) + ).toEqual(['1 run_created', '2 step_created', '3 step_started']); + }); + + test('keeps every claim in a burst of lazy starts', async () => { + // The suspension flush issues its lazy starts at once, each having + // reserved two positions. A second event numbered off the log as this + // world sees it would take the slot the next start in the batch claimed, + // costing every start after the first its claim. + const runId = await newSlotRun(); + const claims = Array.from({ length: 10 }, (_, index) => + slotEventId(FIRST_SLOT + 2 * (index + 1)) + ); + const started = await Promise.all( + claims.map((eventId, index) => + events.create( + runId, + { + eventType: 'step_started', + specVersion: SPEC_VERSION_SLOT_IDENTITY, + correlationId: `step_${index}`, + eventData: { stepName: 'a-step', input: new Uint8Array() }, + }, + { eventId } + ) + ) + ); + expect(started.map((result) => result.event?.eventId)).toEqual(claims); + expect(ascending(await slotsOf(runId))).toEqual( + denseFrom(2 * claims.length + 1) + ); + }); + + test('rejects a claim that leaves no room for the second event', async () => { + // The run's own run_created holds the first slot, so a claim of the second + // means the caller reserved one position for a write that publishes two. + const runId = await newSlotRun(); + await expect( + events.create( + runId, + { + eventType: 'step_started', + specVersion: SPEC_VERSION_SLOT_IDENTITY, + correlationId: 'step_a', + eventData: { stepName: 'a-step', input: new Uint8Array() }, + }, + { eventId: slotEventId(FIRST_SLOT + 1) } + ) + ).rejects.toThrow(/leaves no slot below it/); }); test('numbers events of runs it never created', async () => { From bcd872ec455a6afdddc93e1c83baab995e63388d Mon Sep 17 00:00:00 2001 From: Peter Wielander Date: Fri, 31 Jul 2026 01:00:48 -0700 Subject: [PATCH 13/35] fix(core): floor a turbo run's slot claims above its own positions MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Turbo skips the initial event-log load and replays against an empty snapshot, so the log a claim is numbered from shows neither `run_created` nor the `run_started` still in flight. The floor was only raised from the backgrounded `run_started` response, which is too late to matter: a suspension reserves its whole batch of positions synchronously, so the first batch numbered from an empty log claims the two positions the run already holds, both ops lose their claims, and after the bounded reclaim retries one of them is dropped for good — the run then waits forever on an event that will never be written. Both positions are certain the moment turbo engages, so seed the floor there instead. Co-Authored-By: Claude Opus 5 --- packages/core/src/runtime.test.ts | 44 ++++++++++++++++++++++++++++--- packages/core/src/runtime.ts | 16 +++++++++++ 2 files changed, 56 insertions(+), 4 deletions(-) diff --git a/packages/core/src/runtime.test.ts b/packages/core/src/runtime.test.ts index aafec00075..3a42435b01 100644 --- a/packages/core/src/runtime.test.ts +++ b/packages/core/src/runtime.test.ts @@ -6,7 +6,10 @@ import { } from '@workflow/errors'; import { type Event, + FIRST_SLOT, SPEC_VERSION_CURRENT, + SPEC_VERSION_SLOT_IDENTITY, + slotFromId, type WorkflowRun, } from '@workflow/world'; import { ulid } from 'ulid'; @@ -1472,12 +1475,15 @@ describe('workflowEntrypoint turbo mode', () => { return r; }${xform('workflow')}`; - async function makeRunInput(runId: string) { + async function makeRunInput( + runId: string, + specVersion = SPEC_VERSION_CURRENT + ) { return { input: await dehydrateWorkflowArguments([], runId, undefined, []), deploymentId: 'test-deployment', workflowName: 'workflow', - specVersion: SPEC_VERSION_CURRENT, + specVersion, executionContext: {}, }; } @@ -1493,8 +1499,10 @@ describe('workflowEntrypoint turbo mode', () => { attempt: number; source: string; runStartedGate?: Promise; + specVersion?: typeof SPEC_VERSION_CURRENT; }) { const { runId, attempt, source } = opts; + const specVersion = opts.specVersion ?? SPEC_VERSION_CURRENT; const order = turboOrder; const durable: Event[] = []; let seq = 0; @@ -1513,6 +1521,7 @@ describe('workflowEntrypoint turbo mode', () => { runId, workflowName: 'workflow', status: 'running', + specVersion, input: await dehydrateWorkflowArguments([], runId, undefined, []), createdAt: new Date('2024-01-01T00:00:00.000Z'), updatedAt: new Date('2024-01-01T00:00:00.000Z'), @@ -1558,7 +1567,7 @@ describe('workflowEntrypoint turbo mode', () => { }); setWorld({ - specVersion: SPEC_VERSION_CURRENT, + specVersion, createQueueHandler: vi.fn( (_p: string, handler: (m: unknown, md: unknown) => Promise) => async () => { @@ -1566,7 +1575,7 @@ describe('workflowEntrypoint turbo mode', () => { { runId, requestedAt: new Date('2024-01-01T00:00:00.000Z'), - runInput: await makeRunInput(runId), + runInput: await makeRunInput(runId, specVersion), }, { requestId: 'req_turbo', @@ -1652,6 +1661,33 @@ describe('workflowEntrypoint turbo mode', () => { ); }); + it('claims slots above the run own positions on a first delivery', async () => { + // Turbo replays against an empty snapshot, so the log the claims are + // numbered from cannot show `run_created` or the in-flight `run_started`. + // Both positions are nonetheless taken, and the mocked `run_started` + // response reports no event — the same shape as a World that skips the + // preload — so nothing but the floor seeded at turbo entry keeps the first + // batch of claims off them. + const { handlerPromise, eventsCreate } = await driveTurbo({ + runId: 'wrun_turbo_slots', + attempt: 1, + source: stepAndSleepWorkflow, + specVersion: SPEC_VERSION_SLOT_IDENTITY, + }); + + const res = await handlerPromise; + expect(res.status).toBe(204); + + const claimed = eventsCreate.mock.calls + .map((c) => (c[2] as { eventId?: unknown } | undefined)?.eventId) + .filter((id): id is string => typeof id === 'string'); + // The sleep's `wait_created` is claimed, so there is something to assert on. + expect(claimed.length).toBeGreaterThan(0); + for (const eventId of claimed) { + expect(slotFromId(eventId)).toBeGreaterThan(FIRST_SLOT + 1); + } + }); + it('does not turbo when WORKFLOW_TURBO=0 (parity with the awaited path)', async () => { process.env.WORKFLOW_TURBO = '0'; const { handlerPromise, order } = await driveTurbo({ diff --git a/packages/core/src/runtime.ts b/packages/core/src/runtime.ts index 95d19866a2..45928f36aa 100644 --- a/packages/core/src/runtime.ts +++ b/packages/core/src/runtime.ts @@ -17,6 +17,7 @@ import { } from '@workflow/utils/parse-name'; import { type Event, + FIRST_SLOT, getQueueTopicPrefix, isLegacySpecVersion, ROOT_RUN_ID_ATTRIBUTE, @@ -24,6 +25,7 @@ import { SPEC_VERSION_CURRENT, SPEC_VERSION_SUPPORTS_COMPRESSION, slotFromId, + usesSlotIdentity, WorkflowInvokePayloadSchema, type WorkflowRun, type World, @@ -1055,6 +1057,20 @@ export function workflowEntrypoint( // intentionally truthy here — do not change the load // branches' `if (preloadedEvents)` checks to test length. preloadedEvents = []; + // A slot-numbered run's first two positions are the run's + // own: `run_created` from start(), then the `run_started` + // in flight above. Both are certain before any write of + // this invocation, and turbo replays against the empty + // snapshot skipped just above — so seed the floor with + // them here rather than waiting for the backgrounded + // response to report it. Waiting loses the race: a + // suspension reserves its whole batch of positions + // synchronously, so a batch that starts numbering from an + // empty log claims the two the run already holds and the + // ops holding them lose their claims. + if (usesSlotIdentity(runInput.specVersion)) { + knownSlotFloor = FIRST_SLOT + 1; + } const now = new Date(); workflowRun = { runId, From 84ababeb23cf6ba8c0e1c3b2907284735210127b Mon Sep 17 00:00:00 2001 From: Peter Wielander Date: Fri, 31 Jul 2026 00:23:56 -0700 Subject: [PATCH 14/35] fix(core): reserve a slot for every event a write publishes MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit A lazy inline step start publishes two events: the World also writes the `step_created` the start deferred. That second event takes the slot immediately below the claim, so the claim has to reserve it — the batch hands out slots synchronously, before any of them land, and a second event numbered off the log as the backend sees it lands on the slot the next start in the batch is holding and costs that start its claim. Co-Authored-By: Claude Opus 5 --- packages/core/src/runtime.ts | 13 ++++++++++- packages/core/src/runtime/helpers.test.ts | 27 +++++++++++++++++++++++ packages/core/src/runtime/helpers.ts | 24 ++++++++++++++++++-- 3 files changed, 61 insertions(+), 3 deletions(-) diff --git a/packages/core/src/runtime.ts b/packages/core/src/runtime.ts index 73d4403075..300c2c465c 100644 --- a/packages/core/src/runtime.ts +++ b/packages/core/src/runtime.ts @@ -2337,7 +2337,18 @@ export function workflowEntrypoint( // it is assigned in has to be replay-stable. const eventCreateFence = eventCreateFenceFor( inlineClaimLog, - workflowRun.specVersion + workflowRun.specVersion, + { + // A lazy start publishes two events: the World + // writes the step's deferred `step_created` + // alongside the claim, so the batch has to + // reserve a slot for that one too — otherwise + // it lands on the slot the next start in the + // batch is holding and costs that start its + // claim. + extraEvents: + s.lazyStepInput !== undefined ? 1 : 0, + } ); const run = () => executeStep({ diff --git a/packages/core/src/runtime/helpers.test.ts b/packages/core/src/runtime/helpers.test.ts index a7fd428598..16374fba70 100644 --- a/packages/core/src/runtime/helpers.test.ts +++ b/packages/core/src/runtime/helpers.test.ts @@ -616,6 +616,33 @@ describe('slot bookkeeping', () => { }); }); + it('reserves a slot per extra event and names the top one', () => { + // A lazy inline `step_started` publishes two events: the World also writes + // the `step_created` it deferred, which takes the slot below the claim. + // Reserving it here is what keeps it off the slot the next write of the + // same batch will claim. + const log = toMutableEventLog([slotEvent(1)], null); + expect( + eventCreateFenceFor(log, SPEC_VERSION_SLOT_IDENTITY, { extraEvents: 1 }) + ).toEqual({ eventId: slotEventId(3), maxSlot: 1 }); + expect( + eventCreateFenceFor(log, SPEC_VERSION_SLOT_IDENTITY, { extraEvents: 1 }) + ).toEqual({ eventId: slotEventId(5), maxSlot: 1 }); + // A single-event write in the same batch still gets the next free slot. + expect(eventCreateFenceFor(log, SPEC_VERSION_SLOT_IDENTITY)).toEqual({ + eventId: slotEventId(6), + maxSlot: 1, + }); + }); + + it('burns no slot on an extra event of a ULID-numbered run', () => { + const log = toMutableEventLog([], null); + eventCreateFenceFor(log, SPEC_VERSION_SLOT_IDENTITY - 1, { + extraEvents: 1, + }); + expect(log.reserved).toBe(0); + }); + it('proposes no event id for a ULID-numbered run', () => { // A run whose ids the backend mints must not burn slots either. const log = toMutableEventLog([], null); diff --git a/packages/core/src/runtime/helpers.ts b/packages/core/src/runtime/helpers.ts index 6720f22cd7..407f202635 100644 --- a/packages/core/src/runtime/helpers.ts +++ b/packages/core/src/runtime/helpers.ts @@ -866,13 +866,33 @@ export interface EventCreateFence { * distinct slot per create. `undefined` when the run is fenced neither way, * leaving the create exactly as unfenced as it was before either mechanism * existed. + * + * `extraEvents` is how many events *besides* the one being created this write + * publishes: a lazy inline `step_started` also materializes the `step_created` + * it deferred. Those events take the slots immediately below the claim, so this + * reserves them too and names the top one — a World that writes a pair + * derives the lower id from the one it was given. + * + * The reservation has to happen here rather than at the World or its backend. + * Slots are handed out for a whole concurrent batch synchronously, before any of + * it lands, so a second event numbered off the log as the backend sees it would + * take the slot already promised to the next write in the batch — and every + * write after the first in a fan-out would lose its claim. */ export function eventCreateFenceFor( log: MutableEventLog, - specVersion: number | undefined + specVersion: number | undefined, + options?: { extraEvents?: number } ): EventCreateFence | undefined { if (usesSlotIdentity(specVersion)) { - return { eventId: slotEventId(reserveSlot(log)), maxSlot: log.maxSlot }; + const maxSlot = log.maxSlot; + // The extra events sit below the one being created, matching the order a + // reader expects (a step is created before it starts) — so their slots are + // reserved first and the claim names the last of the run. + for (let i = 0; i < (options?.extraEvents ?? 0); i++) { + reserveSlot(log); + } + return { eventId: slotEventId(reserveSlot(log)), maxSlot }; } const stateUpdatedAt = stateUpdatedAtForCreate(log.events, specVersion); return stateUpdatedAt !== undefined ? { stateUpdatedAt } : undefined; From b4a6b2e875e400dc000fdb655059bc942294f103 Mon Sep 17 00:00:00 2001 From: Peter Wielander Date: Thu, 30 Jul 2026 23:20:10 -0700 Subject: [PATCH 15/35] fix(world-local): re-probe a lost position instead of conflicting MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit A writer with no event log — a step completion reporting in, a hook being received, a cancellation from an API call — has nothing to reconcile when the position it allocated turns out to be taken, so surfacing a slot conflict to it strands the caller with an error it cannot act on. Two world instances sharing a data directory keep independent books, which makes that the common case rather than a rarity. The publish is now one attempt in a loop: an allocated position that loses tops the book up from disk, backs off with full jitter, and takes the next free one, within a 30s budget after which the write fails retryably. A position the caller claimed still conflicts on the first loss — the claim asserts a complete log, so only the caller can resolve it. The contention profile moves to @workflow/world so the Local and Postgres Worlds share one definition. --- .../world-local/src/storage/events-storage.ts | 301 +++++++++++------- .../src/storage/slot-identity.test.ts | 39 ++- packages/world-local/src/storage/slots.ts | 28 ++ packages/world-postgres/src/slots.ts | 29 +- packages/world/src/index.ts | 4 + packages/world/src/slot-identity.ts | 25 ++ 6 files changed, 273 insertions(+), 153 deletions(-) diff --git a/packages/world-local/src/storage/events-storage.ts b/packages/world-local/src/storage/events-storage.ts index 98e705e98d..72c8757a46 100644 --- a/packages/world-local/src/storage/events-storage.ts +++ b/packages/world-local/src/storage/events-storage.ts @@ -37,9 +37,11 @@ import { requiresNewerWorld, SPEC_VERSION_CURRENT, SPEC_VERSION_MAX_SUPPORTED, + SLOT_RETRY_BUDGET_MS, StepSchema, slotEventId, slotFromId, + slotRetryDelay, ulidToDate, usesSlotIdentity, validateAttributeChanges, @@ -1009,6 +1011,13 @@ export function createEventsStorage( // ============================================================ // EVENT ID: the caller's slot claim, an allocated slot, or a ULID // ============================================================ + // A run's own `run_created` owns the first slot — provably, since + // nothing precedes it — so every other event allocates above it, even + // when that event is the first to arrive here. + const allocationFloor = + data.eventType === 'run_created' + ? RUN_CREATED_SLOT + : RUN_CREATED_SLOT + 1; // A slot-numbered run's ids name positions in its log, so an id is // either claimed by a caller that holds the log (and is therefore // asserting the log is complete up to that position) or allocated here @@ -1042,15 +1051,7 @@ export function createEventsStorage( } } else if (slotMode) { reservedRunId = effectiveRunId; - // A run's own `run_created` owns the first slot — provably, since - // nothing precedes it — so every other event allocates above it, even - // when that event is the first to arrive here. - const slot = await slots.reserve( - effectiveRunId, - data.eventType === 'run_created' - ? RUN_CREATED_SLOT - : RUN_CREATED_SLOT + 1 - ); + const slot = await slots.reserve(effectiveRunId, allocationFloor); reserved.add(slot); eventId = slotEventId(slot); } @@ -2457,120 +2458,172 @@ export function createEventsStorage( throw new HookNotFoundError(data.correlationId); } - const compositeKey = `${effectiveRunId}-${eventId}`; - const eventPath = taggedPath(basedir, 'events', compositeKey, tag); - // Capture the serialized payload before the write's `await` so the - // cached snapshot can't observe a later mutation (see - // rememberStoredEvent). - const serializedEvent = JSON.stringify(event, jsonReplacer, 2); - - // Cross-process terminal-run guard for `hook_received`. A terminal - // transition (run_completed / run_failed / run_cancelled) in ANY - // process (1) publishes a durable `runTerminalMarkerPath` marker and - // (2) reaps the run's staged hook_received events, both BEFORE it - // writes the terminal run state or appends its terminal event (see - // the terminal-transition block earlier in this function). In-memory - // locks cannot close the shared-filesystem race this backend - // explicitly supports, and a published event file is immediately - // visible to `events.list()` in other processes — so it can never be - // "rolled back" after the fact. Instead, the event stays INVISIBLE - // to readers until a single atomic filesystem operation decides its - // fate: - // - // 1. (fast path) reject if the run is already terminal — by - // marker, or by run state for runs that predate the marker — - // so the common case never creates a file. - // 2. STAGE the event at a non-reader-visible path under `.locks`. - // 3. re-CHECK the terminal marker; reject if present. - // 4. PROMOTE the staged file into `events/` with an atomic hard - // link; reject if the staged file was reaped (`'missing'`). - // - // Correctness: the reap's `unlink` and step 4's `link` target the - // same staged file, so the filesystem serializes them — exactly one - // wins. If the link wins, the event was reader-visible before the - // reap completed, and therefore before the terminal state and - // terminal event were written: acceptance happened-before the - // termination and legitimately precedes it. If the unlink wins, - // promotion fails and the event is never visible to any reader — - // there is nothing to roll back. A resume that stages after the - // reap has passed necessarily stages after the marker was - // committed, so step 3 rejects it. Rejections before step 4 unlink - // a file no reader can see. - let eventPublished: boolean; - if (data.eventType === 'hook_received') { - // Step 1: fast path. The marker is the authoritative durable - // signal; the run-state read additionally rejects runs whose - // terminal state was written without a marker (e.g. runs that - // terminated on an older storage version). - const terminalByMarker = await isRunTerminalCommitted( - basedir, - effectiveRunId, - tag - ); - const runNow = terminalByMarker - ? null - : await readJSONWithFallback( - basedir, - 'runs', - effectiveRunId, - WorkflowRunSchema, - tag - ); - if ( - terminalByMarker || - (runNow && isTerminalWorkflowRunStatus(runNow.status)) - ) { - throw new RunExpiredError( - `Workflow run "${effectiveRunId}" is already in a terminal state` - ); - } - - const stagedPath = pendingHookEventPath( - basedir, - effectiveRunId, - eventId, - tag - ); - const staged = await writeExclusive(stagedPath, serializedEvent); - if (!staged) { - // For a ULID-numbered run the eventId is freshly generated, so - // its staging path can only be occupied by a previous crashed - // attempt of this very event, which never promoted. A - // slot-numbered run can also collide here with another instance - // that allocated the same slot from its own book. Either way the - // event is not reader-visible, so there is no delta to hand back - // and nothing for the caller to merge: surface the same conflict - // shape as a visible-path collision, and let the allocator - // re-probe on the retry. - throw new EntityConflictError( - `Event "${eventId}" already exists for run "${effectiveRunId}"` + /** + * One attempt at publishing the event at the position `eventId` + * currently names: `true` when this call made it reader-visible, + * `false` when the position was already taken. What a loss means is the + * loop's decision — a position this world allocated is simply retried + * one higher, a position the caller claimed is a conflict it has to + * resolve. + */ + async function publishOnce(): Promise { + // Cross-process terminal-run guard for `hook_received`. A terminal + // transition (run_completed / run_failed / run_cancelled) in ANY + // process (1) publishes a durable `runTerminalMarkerPath` marker and + // (2) reaps the run's staged hook_received events, both BEFORE it + // writes the terminal run state or appends its terminal event (see + // the terminal-transition block earlier in this function). In-memory + // locks cannot close the shared-filesystem race this backend + // explicitly supports, and a published event file is immediately + // visible to `events.list()` in other processes — so it can never be + // "rolled back" after the fact. Instead, the event stays INVISIBLE + // to readers until a single atomic filesystem operation decides its + // fate: + // + // 1. (fast path) reject if the run is already terminal — by + // marker, or by run state for runs that predate the marker — + // so the common case never creates a file. + // 2. STAGE the event at a non-reader-visible path under `.locks`. + // 3. re-CHECK the terminal marker; reject if present. + // 4. PROMOTE the staged file into `events/` with an atomic hard + // link; reject if the staged file was reaped (`'missing'`). + // + // Correctness: the reap's `unlink` and step 4's `link` target the + // same staged file, so the filesystem serializes them — exactly one + // wins. If the link wins, the event was reader-visible before the + // reap completed, and therefore before the terminal state and + // terminal event were written: acceptance happened-before the + // termination and legitimately precedes it. If the unlink wins, + // promotion fails and the event is never visible to any reader — + // there is nothing to roll back. A resume that stages after the + // reap has passed necessarily stages after the marker was + // committed, so step 3 rejects it. Rejections before step 4 unlink + // a file no reader can see. + if (data.eventType === 'hook_received') { + // Step 1: fast path. The marker is the authoritative durable + // signal; the run-state read additionally rejects runs whose + // terminal state was written without a marker (e.g. runs that + // terminated on an older storage version). + const terminalByMarker = await isRunTerminalCommitted( + basedir, + effectiveRunId, + tag ); - } - try { - if (await isRunTerminalCommitted(basedir, effectiveRunId, tag)) { + const runNow = terminalByMarker + ? null + : await readJSONWithFallback( + basedir, + 'runs', + effectiveRunId, + WorkflowRunSchema, + tag + ); + if ( + terminalByMarker || + (runNow && isTerminalWorkflowRunStatus(runNow.status)) + ) { throw new RunExpiredError( `Workflow run "${effectiveRunId}" is already in a terminal state` ); } - const promoted = await promoteExclusive(stagedPath, eventPath); - if (promoted === 'missing') { - // A terminal transition reaped the staged file between the - // check and the link — the atomic loss of the arbitration. - throw new RunExpiredError( - `Workflow run "${effectiveRunId}" is already in a terminal state` + + const stagedPath = pendingHookEventPath( + basedir, + effectiveRunId, + eventId, + tag + ); + const staged = await writeExclusive(stagedPath, serializedEvent); + if (!staged) { + // For a ULID-numbered run the eventId is freshly generated, so + // its staging path can only be occupied by a previous crashed + // attempt of this very event, which never promoted. A + // slot-numbered run can also collide here with another instance + // that allocated the same slot from its own book. Either way the + // event is not reader-visible, so there is no delta to hand back + // and nothing for the caller to merge: surface the same conflict + // shape as a visible-path collision — or, when this world + // allocated the position itself, let the loop below re-probe and + // take the next one. + if (reallocatesSlot) { + return false; + } + throw new EntityConflictError( + `Event "${eventId}" already exists for run "${effectiveRunId}"` ); } - eventPublished = promoted === 'linked'; - } finally { - // The staged path is not reader-visible; removing it is pure - // cleanup on every outcome (already gone when reaped). - await deleteJSON(stagedPath).catch(() => {}); + try { + if (await isRunTerminalCommitted(basedir, effectiveRunId, tag)) { + throw new RunExpiredError( + `Workflow run "${effectiveRunId}" is already in a terminal state` + ); + } + const promoted = await promoteExclusive(stagedPath, eventPath); + if (promoted === 'missing') { + // A terminal transition reaped the staged file between the + // check and the link — the atomic loss of the arbitration. + throw new RunExpiredError( + `Workflow run "${effectiveRunId}" is already in a terminal state` + ); + } + return promoted === 'linked'; + } finally { + // The staged path is not reader-visible; removing it is pure + // cleanup on every outcome (already gone when reaped). + await deleteJSON(stagedPath).catch(() => {}); + } } - } else { - eventPublished = await writeExclusive(eventPath, serializedEvent); + return await writeExclusive(eventPath, serializedEvent); } - if (!eventPublished) { + // A write that allocated its own position may take the next free one + // when it loses: nothing outside this world named the slot, so which + // position the event lands on is this world's business, and the caller + // — a step reporting its completion, a hook being received — has no log + // to reconcile. A write whose position the *caller* claimed may not: + // the claim asserts a log complete up to that position, so losing it + // means that log is stale and only the caller can resolve it. + const reallocatesSlot = slotMode && params?.eventId === undefined; + const slotDeadline = Date.now() + SLOT_RETRY_BUDGET_MS; + let compositeKey = ''; + let eventPath = ''; + let serializedEvent = ''; + let eventPublished = false; + + for (let round = 0; ; round++) { + compositeKey = `${effectiveRunId}-${eventId}`; + eventPath = taggedPath(basedir, 'events', compositeKey, tag); + // Capture the serialized payload before the write's `await` so the + // cached snapshot can't observe a later mutation (see + // rememberStoredEvent). + serializedEvent = JSON.stringify(event, jsonReplacer, 2); + eventPublished = await publishOnce(); + if (eventPublished) { + break; + } + if (reallocatesSlot && Date.now() < slotDeadline) { + // The position is someone else's — either published there or + // staged for it. Record that, top the book up from disk, and try + // again one position higher rather than surfacing a conflict the + // caller cannot act on. Re-reading rather than incrementing keeps + // the log dense: every round at least one writer wins, so the + // search never runs away from the log it is filling. + const lost = slotFromId(eventId); + if (lost !== undefined) { + reserved.delete(lost); + } + slots.observe(effectiveRunId, eventId); + await slots.refresh(effectiveRunId); + await new Promise((resolve) => + setTimeout(resolve, slotRetryDelay(round)) + ); + const slot = await slots.reserve(effectiveRunId, allocationFloor); + reservedRunId = effectiveRunId; + reserved.add(slot); + eventId = slotEventId(slot); + event = { ...event, eventId }; + continue; + } // For `hook_created`, losing the event publish means the // event was already committed at this exact (canonical) // path. The original publisher may have crashed between @@ -2591,13 +2644,23 @@ export function createEventsStorage( tag ); } + if (reallocatesSlot) { + // Out of budget: every position this writer tried was taken by + // someone else. Surfacing it as a 503 puts the whole operation + // back on the queue rather than stalling the run here. + throw new WorkflowWorldError( + `Could not place an event in run "${effectiveRunId}" within ${SLOT_RETRY_BUDGET_MS}ms of contention`, + { status: 503 } + ); + } if (slotMode) { - // Losing a slot means someone else's event occupies this position, - // so the log this event was derived from is missing at least that - // event — the whole proposed event is stale, not just its id. Hand - // back what the caller is missing so it can merge, replay and - // re-propose, and forget the run's book so the next allocation - // re-reads the log this instance evidently does not have. + // Losing a claimed slot means someone else's event occupies this + // position, so the log this event was derived from is missing at + // least that event — the whole proposed event is stale, not just + // its id. Hand back what the caller is missing so it can merge, + // replay and re-propose, and forget the run's book so the next + // allocation re-reads the log this instance evidently does not + // have. // // Reaching here means the slot was taken *after* the pre-check at // the claim site, so the entity this event was going to describe diff --git a/packages/world-local/src/storage/slot-identity.test.ts b/packages/world-local/src/storage/slot-identity.test.ts index 00d8b71cce..f936b55718 100644 --- a/packages/world-local/src/storage/slot-identity.test.ts +++ b/packages/world-local/src/storage/slot-identity.test.ts @@ -249,12 +249,31 @@ describe('conflict', () => { ).toEqual([slotEventId(3)]); }); - it('conflicts across two storage instances sharing a directory', async () => { + it('conflicts when another instance takes a claimed slot', async () => { // Two instances keep independent books, so the exclusive write — not the - // book — is what decides who owns a slot. + // book — is what decides who owns a slot. A claim asserts a complete log, + // so its loser has to reload rather than move over. const runId = await newSlotRun(); const other = createStorage(testDir); - const [first, second] = await Promise.allSettled([ + await other.events.create(runId, { + eventType: 'step_created', + specVersion: SPEC_VERSION_SLOT_IDENTITY, + correlationId: 'step_b', + eventData: { stepName: 'b-step', input: new Uint8Array() }, + }); + await expect(createStep(runId, 'step_a', slotEventId(2))).rejects.toThrow( + SlotConflictError + ); + await expect(slotsOf(runId)).resolves.toEqual([1, 2]); + }); + + it('reallocates around another instance holding the slot it picked', async () => { + // Neither writer holds a log, so neither has anything to reconcile: the + // loser takes the next free position instead of surfacing a conflict its + // caller could not act on. + const runId = await newSlotRun(); + const other = createStorage(testDir); + const outcomes = await Promise.allSettled([ storage.events.create(runId, { eventType: 'step_created', specVersion: SPEC_VERSION_SLOT_IDENTITY, @@ -268,14 +287,10 @@ describe('conflict', () => { eventData: { stepName: 'b-step', input: new Uint8Array() }, }), ]); - const outcomes = [first, second]; - expect(outcomes.filter((o) => o.status === 'fulfilled')).toHaveLength(1); - const rejection = outcomes.find((o) => o.status === 'rejected'); - expect( - SlotConflictError.is( - (rejection as PromiseRejectedResult | undefined)?.reason - ) - ).toBe(true); - await expect(slotsOf(runId)).resolves.toEqual([1, 2]); + expect(outcomes.map((outcome) => outcome.status)).toEqual([ + 'fulfilled', + 'fulfilled', + ]); + await expect(slotsOf(runId)).resolves.toEqual([1, 2, 3]); }); }); diff --git a/packages/world-local/src/storage/slots.ts b/packages/world-local/src/storage/slots.ts index c6b2f017f6..632e31c1ee 100644 --- a/packages/world-local/src/storage/slots.ts +++ b/packages/world-local/src/storage/slots.ts @@ -96,6 +96,16 @@ export interface SlotBook { release(runId: string, slot: number): void; /** Records a published event id, so it is never handed out again. */ observe(runId: string, eventId: string): void; + /** + * Merges the run's published positions from disk into the book kept for it, + * leaving the reservations other writers in this instance still hold. + * + * A writer whose publish lost its position calls this before trying again: + * the book is demonstrably behind another instance's writes, and dropping it + * wholesale ({@link SlotBook.forget}) would hand a sibling's outstanding + * position to the next caller and cost that sibling its own publish. + */ + refresh(runId: string): Promise; /** Drops what is cached for `runId`, so the next reservation re-reads disk. */ forget(runId: string): void; /** Drops everything cached (the data directory was cleared out from under us). */ @@ -227,6 +237,24 @@ export function createSlotBook(basedir: string, tag?: string): SlotBook { book.outstanding.delete(slot); }, + async refresh(runId) { + const book = books.get(runId); + if (!book) { + // Nothing cached to correct; the next reservation seeds from disk. + return; + } + for (const eventId of await listRunEventIds(basedir, runId, tag)) { + const slot = slotFromId(eventId); + if (slot !== undefined) { + book.written.add(slot); + book.outstanding.delete(slot); + } + } + // Positions released while this scan ran may sit below where the search + // had reached, and they are free again. + book.searchFrom = FIRST_SLOT; + }, + forget(runId) { modes.delete(runId); books.delete(runId); diff --git a/packages/world-postgres/src/slots.ts b/packages/world-postgres/src/slots.ts index 89435810a9..2f30ad9f31 100644 --- a/packages/world-postgres/src/slots.ts +++ b/packages/world-postgres/src/slots.ts @@ -14,7 +14,13 @@ */ import { WorkflowWorldError } from '@workflow/errors'; -import { FIRST_SLOT, slotEventId, slotFromId } from '@workflow/world'; +import { + FIRST_SLOT, + SLOT_RETRY_BUDGET_MS, + slotEventId, + slotFromId, + slotRetryDelay, +} from '@workflow/world'; import { and, desc, eq } from 'drizzle-orm'; import { type Drizzle, Schema } from './drizzle/index.js'; @@ -26,19 +32,6 @@ import { type Drizzle, Schema } from './drizzle/index.js'; */ export const RUN_CREATED_SLOT = FIRST_SLOT; -/** First backoff after losing a position; doubled each round. */ -export const SLOT_RETRY_BASE_MS = 5; - -/** Ceiling for a single backoff, so a contended run keeps making attempts. */ -export const SLOT_RETRY_MAX_DELAY_MS = 250; - -/** - * How long a writer keeps looking for a free position before giving up. - * Exhausting it surfaces as a 503, so the caller — in practice a queue - * delivery — retries the whole operation instead of the run stalling on it. - */ -export const SLOT_RETRY_BUDGET_MS = 30_000; - /** Postgres unique-violation code. */ const UNIQUE_VIOLATION = '23505'; @@ -104,14 +97,6 @@ export async function eventExists( return row !== undefined; } -/** Full jitter over an exponentially growing, capped window. */ -export function slotRetryDelay(round: number): number { - return ( - Math.random() * - Math.min(SLOT_RETRY_BASE_MS * 2 ** round, SLOT_RETRY_MAX_DELAY_MS) - ); -} - /** The event ids a single create publishes. */ export interface EventIds { /** diff --git a/packages/world/src/index.ts b/packages/world/src/index.ts index 5a8c185f2a..1ecae0457d 100644 --- a/packages/world/src/index.ts +++ b/packages/world/src/index.ts @@ -106,9 +106,13 @@ export { isSlotId, maxSlotOf, SLOT_ID_WIDTH, + SLOT_RETRY_BASE_MS, + SLOT_RETRY_BUDGET_MS, + SLOT_RETRY_MAX_DELAY_MS, slotEventId, slotFromId, slotIdBody, + slotRetryDelay, } from './slot-identity.js'; export type { SpecVersion } from './spec-version.js'; export { diff --git a/packages/world/src/slot-identity.ts b/packages/world/src/slot-identity.ts index d9ff11b65f..d761f91452 100644 --- a/packages/world/src/slot-identity.ts +++ b/packages/world/src/slot-identity.ts @@ -67,6 +67,31 @@ export function slotEventId(slot: number): string { return `evnt_${slotIdBody(slot)}`; } +/** First backoff after losing a position; doubled each round. */ +export const SLOT_RETRY_BASE_MS = 5; + +/** Ceiling for a single backoff, so a contended run keeps making attempts. */ +export const SLOT_RETRY_MAX_DELAY_MS = 250; + +/** + * How long a writer that allocates its own position keeps looking for a free + * one before giving up. Exhausting it is a retryable failure for the caller — + * in practice a queue delivery — rather than something the run stalls on. + */ +export const SLOT_RETRY_BUDGET_MS = 30_000; + +/** + * Full jitter over an exponentially growing, capped window. Shared by every + * world that allocates positions, so contention behaves the same wherever a run + * is stored. + */ +export function slotRetryDelay(round: number): number { + return ( + Math.random() * + Math.min(SLOT_RETRY_BASE_MS * 2 ** round, SLOT_RETRY_MAX_DELAY_MS) + ); +} + /** * The highest slot named by any of `events`, or 0 when none is slot-numbered. * From ca88d214f0d0295d379a34f32a6d8a1a21453c62 Mon Sep 17 00:00:00 2001 From: Peter Wielander Date: Fri, 31 Jul 2026 00:37:37 -0700 Subject: [PATCH 16/35] fix(worlds): number the deferred step_created below the claim it rides with A lazy start publishes two events, and only the caller knows which positions are free: it hands out a whole batch's slots synchronously, before any of them land. So a claim names the top of the pair and the deferred `step_created` takes the position immediately below it, which the caller reserved for exactly that. A start that claimed nothing has both positions allocated here instead, and a claim with no room below it is rejected rather than allowed to overwrite the run's creation. Co-Authored-By: Claude Opus 5 --- .../world-local/src/storage/events-storage.ts | 82 +++++++++++++++---- .../src/storage/slot-identity.test.ts | 78 ++++++++++++++++++ packages/world-postgres/src/slots.ts | 44 +++++++--- packages/world-postgres/src/storage.ts | 9 +- .../world-postgres/test/slot-identity.test.ts | 63 ++++++++++++-- 5 files changed, 235 insertions(+), 41 deletions(-) diff --git a/packages/world-local/src/storage/events-storage.ts b/packages/world-local/src/storage/events-storage.ts index 72c8757a46..558d5f7d03 100644 --- a/packages/world-local/src/storage/events-storage.ts +++ b/packages/world-local/src/storage/events-storage.ts @@ -1008,6 +1008,17 @@ export function createEventsStorage( } } + // Lazy step start: a step_started carrying step-creation data + // (stepName + input) is allowed to arrive with no prior step_created + // — it creates the step on the fly (see the materialization block + // below). This mirrors the resilient run_started path. Detect it here + // so the second event it publishes can be numbered alongside the + // first, the entity-creation terminal-run guard treats it like a + // creation, and the "step must exist" ordering guard doesn't reject it. + const createsChildEntity = isChildEntityCreationEvent(data); + const lazyStepStart = + createsChildEntity && data.eventType === 'step_started'; + // ============================================================ // EVENT ID: the caller's slot claim, an allocated slot, or a ULID // ============================================================ @@ -1023,6 +1034,11 @@ export function createEventsStorage( // asserting the log is complete up to that position) or allocated here // for a caller that has no log — a step completion reporting in, a // cancellation from an API call. + // + // The position of the second event a lazy start publishes, when this + // one publishes two. Consumed by the materialization below; released + // again if that block turns out not to need it. + let companionSlot: number | undefined; if (params?.eventId !== undefined) { const claimedSlot = slotFromId(params.eventId); if (!slotMode) { @@ -1041,13 +1057,38 @@ export function createEventsStorage( reservedRunId = effectiveRunId; reserved.add(claimedSlot); slots.claim(effectiveRunId, claimedSlot); - if (await slots.isWritten(effectiveRunId, claimedSlot)) { - // Reject a doomed claim before the materialization below creates - // the step, hook or wait this event will now never accompany. A - // caller that re-proposes at the next slot would otherwise - // collide with its own orphan and read that as "my write already - // landed". See SlotBook.isWritten. - throw await slotConflict(effectiveRunId, eventId, params); + // One request, two events: a lazy start also publishes the + // `step_created` it deferred. A claim names the *top* of the pair, + // so the second event takes the slot immediately below it — the + // caller reserved both positions and named only one, which is what + // keeps the pair from landing on a position another write in the + // same batch is already holding. + if (lazyStepStart) { + companionSlot = claimedSlot - 1; + if (companionSlot < RUN_CREATED_SLOT + 1) { + throw new WorkflowWorldError( + `Event id "${params.eventId}" leaves no slot below it in run "${effectiveRunId}" for the "step_created" published alongside it`, + { status: 400 } + ); + } + reserved.add(companionSlot); + slots.claim(effectiveRunId, companionSlot); + } + // Reject a doomed claim before the materialization below creates + // the step, hook or wait this event will now never accompany. A + // caller that re-proposes at the next slot would otherwise + // collide with its own orphan and read that as "my write already + // landed". See SlotBook.isWritten. + for (const slot of companionSlot === undefined + ? [claimedSlot] + : [companionSlot, claimedSlot]) { + if (await slots.isWritten(effectiveRunId, slot)) { + throw await slotConflict( + effectiveRunId, + slotEventId(slot), + params + ); + } } } else if (slotMode) { reservedRunId = effectiveRunId; @@ -1060,16 +1101,6 @@ export function createEventsStorage( // VALIDATION: Terminal state and event ordering checks // ============================================================ - // Lazy step start: a step_started carrying step-creation data - // (stepName + input) is allowed to arrive with no prior step_created - // — it creates the step on the fly (see the materialization block - // below). This mirrors the resilient run_started path. Detect it here - // so the entity-creation terminal-run guard treats it like a creation - // and the "step must exist" ordering guard doesn't reject it. - const createsChildEntity = isChildEntityCreationEvent(data); - const lazyStepStart = - createsChildEntity && data.eventType === 'step_started'; - // Run terminal state validation if (currentRun && isTerminalWorkflowRunStatus(currentRun.status)) { // Idempotent operation: run_cancelled on already cancelled run is allowed @@ -1724,7 +1755,13 @@ export function createEventsStorage( // run_started → run_created precedent in this file. let stepCreatedEventId = `evnt_${monotonicUlid()}`; if (slotMode) { - const slot = await slots.reserve(effectiveRunId); + // A claimed start numbers this event one below its own + // position, which the caller reserved for exactly this. A start + // that allocated takes the next free slot instead: nothing + // outside this world named either position. + const slot = + companionSlot ?? (await slots.reserve(effectiveRunId)); + companionSlot = undefined; reserved.add(slot); stepCreatedEventId = slotEventId(slot); } @@ -2679,6 +2716,15 @@ export function createEventsStorage( // replay can serve it without rereading from disk. rememberStoredEvent(event, eventPath, serializedEvent); slots.observe(effectiveRunId, eventId); + if (companionSlot !== undefined) { + // A start that carried creation data for a step that already existed + // synthesized no `step_created`, so the position below it went + // unused. Hand it back instead of leaving it outstanding for the life + // of the process, where it would block the allocator from ever + // filling that position. + reserved.delete(companionSlot); + slots.release(effectiveRunId, companionSlot); + } // Write the hook entity ONLY now that the event publish has // committed. Doing this earlier (in the `hook_created` diff --git a/packages/world-local/src/storage/slot-identity.test.ts b/packages/world-local/src/storage/slot-identity.test.ts index f936b55718..885126e5fa 100644 --- a/packages/world-local/src/storage/slot-identity.test.ts +++ b/packages/world-local/src/storage/slot-identity.test.ts @@ -147,6 +147,84 @@ describe('numbering', () => { }); }); +/** + * A lazy step start: a `step_started` carrying the step's creation data, which + * the world materializes into a step plus the `step_created` event the caller + * deferred — one request, two events. + */ +async function startStepLazily( + runId: string, + stepId: string, + eventId?: string +): Promise { + const result = await storage.events.create( + runId, + { + eventType: 'step_started', + specVersion: SPEC_VERSION_SLOT_IDENTITY, + correlationId: stepId, + eventData: { stepName: 'a-step', input: new Uint8Array(), attempt: 0 }, + }, + eventId === undefined ? undefined : { eventId } + ); + if (!result.event) { + throw new Error('Expected an event'); + } + return result.event.eventId; +} + +describe('a write that publishes two events', () => { + it('numbers the deferred step_created below the claim', async () => { + // The caller reserves both positions and names only the top one, so the + // pair is fixed before either lands — which is what keeps it off the slot + // the next write of the same batch is holding. + const runId = await newSlotRun(); + const startedEventId = await startStepLazily( + runId, + 'step_a', + slotEventId(3) + ); + expect(startedEventId).toBe(slotEventId(3)); + await expect(slotsOf(runId)).resolves.toEqual([1, 2, 3]); + }); + + it('allocates both positions for a start that claims neither', async () => { + const runId = await newSlotRun(); + await startStepLazily(runId, 'step_a'); + await expect(slotsOf(runId)).resolves.toEqual([1, 2, 3]); + }); + + it('keeps every claim in a burst of lazy starts', async () => { + // The suspension flush issues its lazy starts at once, each having reserved + // two positions. A second event numbered off the log as this world sees it + // would take the slot the next start in the batch claimed, and cost every + // start after the first its claim — collapsing the fan-out to one step. + const runId = await newSlotRun(); + const claims = Array.from({ length: 10 }, (_, index) => + slotEventId(FIRST_SLOT + 2 * (index + 1)) + ); + const ids = await Promise.all( + claims.map((eventId, index) => + startStepLazily(runId, `step_${index}`, eventId) + ) + ); + expect(ids).toEqual(claims); + const slots = await slotsOf(runId); + expect([...slots].sort((a, b) => a - b)).toEqual( + Array.from({ length: 2 * claims.length + 1 }, (_, i) => FIRST_SLOT + i) + ); + }); + + it('rejects a claim that leaves no room for the second event', async () => { + // The run's own run_created holds the first slot, so a claim of the second + // means the caller reserved one position for a write that publishes two. + const runId = await newSlotRun(); + await expect( + startStepLazily(runId, 'step_a', slotEventId(FIRST_SLOT + 1)) + ).rejects.toThrow(/leaves no slot below it/); + }); +}); + describe('mode is pinned to the run', () => { it('rejects a slot id claimed on a ULID-numbered run', async () => { const created = await storage.events.create(null, { diff --git a/packages/world-postgres/src/slots.ts b/packages/world-postgres/src/slots.ts index 2f30ad9f31..cd0a54f6f5 100644 --- a/packages/world-postgres/src/slots.ts +++ b/packages/world-postgres/src/slots.ts @@ -107,9 +107,12 @@ export interface EventIds { primary: () => string; /** * An additional event written in the same breath — the synthetic - * `step_created` of a lazy step start. Its position is allocated here even - * when the caller named its own for the primary event, because a caller that - * defers a `step_created` cannot know it will be synthesized. + * `step_created` of a lazy step start. + * + * A claim names the *top* of the pair, so this event takes the position + * immediately below it: the caller reserved both and named one. Numbering it + * off the log instead would hand it a position another write of the same + * concurrent batch is already holding, and cost that write its claim. */ extra: () => Promise; } @@ -178,23 +181,40 @@ export async function placeEvent( options.claimedSlot === undefined ? slotEventId(await take()) : slotEventId(options.claimedSlot); + /** Positions the caller named, which are the caller's to resolve. */ + const claimed = options.claimedSlot === undefined ? [] : [primary]; try { return await options.write({ primary: () => primary, - extra: async () => slotEventId(await take()), + extra: async () => { + if (options.claimedSlot === undefined) { + return slotEventId(await take()); + } + const slot = options.claimedSlot - 1; + if (slot <= FIRST_SLOT) { + // The run's own `run_created` holds the first slot, so a claim of + // the second leaves nowhere for a second event to go: the caller + // reserved one position for a write that publishes two. + throw new WorkflowWorldError( + `Event id "${primary}" leaves no slot below it in run "${runId}" for the second event published alongside it`, + { status: 400 } + ); + } + const id = slotEventId(slot); + claimed.push(id); + return id; + }, }); } catch (error) { if (!isEventKeyViolation(error)) { throw error; } - // A claimed write can also lose on its extra event's position, which is - // this world's to reallocate — only a claim that is itself taken is the - // caller's problem. - if ( - options.claimedSlot !== undefined && - (await eventExists(drizzle, runId, primary)) - ) { - throw await options.onClaimTaken(); + // Only a position the caller named is the caller's problem; one this + // world allocated is reallocated below without ever surfacing. + for (const id of claimed) { + if (await eventExists(drizzle, runId, id)) { + throw await options.onClaimTaken(); + } } if (Date.now() >= deadline) { throw new WorkflowWorldError( diff --git a/packages/world-postgres/src/storage.ts b/packages/world-postgres/src/storage.ts index 878f39758b..ab752f7de5 100644 --- a/packages/world-postgres/src/storage.ts +++ b/packages/world-postgres/src/storage.ts @@ -1411,11 +1411,10 @@ export function createEventsStorage(drizzle: Drizzle): Storage['events'] { // side of the materialization would, hence the shared // transaction. // - // It takes a position of its own, which on a claimed write is - // necessarily one this world allocated: a caller that defers a - // step_created cannot know it will be synthesized, so it never - // claims a slot for it. Losing that position rolls the transaction - // back for a retry, so the insert must not swallow the collision. + // It takes a position of its own — the one below the claim on a + // claimed write, the next free one otherwise. Losing that position + // rolls the transaction back for a retry, so the insert must not + // swallow the collision. const stepCreatedEventId = await ids.extra(); const insertStepCreated = tx.insert(events).values({ runId: effectiveRunId, diff --git a/packages/world-postgres/test/slot-identity.test.ts b/packages/world-postgres/test/slot-identity.test.ts index d8b0578a7d..8e3f04c8a4 100644 --- a/packages/world-postgres/test/slot-identity.test.ts +++ b/packages/world-postgres/test/slot-identity.test.ts @@ -173,9 +173,10 @@ describe('Slot identity (Postgres integration)', () => { ).toEqual(['1 run_created', '2 step_started', '3 step_created']); }); - test('allocates the deferred step_created above a claimed slot', async () => { - // A caller that defers a step_created cannot know it will be synthesized, - // so it never claims a slot for it — and the slot it did claim is its own. + test('numbers the deferred step_created below a claimed slot', async () => { + // A claim names the top of the pair: the caller reserved both positions + // before either landed, which is what keeps the second event off the slot + // the next write of the same batch claimed. const runId = await newSlotRun(); const started = await events.create( runId, @@ -185,10 +186,60 @@ describe('Slot identity (Postgres integration)', () => { correlationId: 'step_a', eventData: { stepName: 'a-step', input: new Uint8Array() }, }, - { eventId: slotEventId(2) } + { eventId: slotEventId(3) } ); - expect(started.event?.eventId).toBe(slotEventId(2)); - expect(ascending(await slotsOf(runId))).toEqual(denseFrom(3)); + expect(started.event?.eventId).toBe(slotEventId(3)); + const { data } = await eventsOf(runId); + expect( + data.map((event) => `${slotFromId(event.eventId)} ${event.eventType}`) + ).toEqual(['1 run_created', '2 step_created', '3 step_started']); + }); + + test('keeps every claim in a burst of lazy starts', async () => { + // The suspension flush issues its lazy starts at once, each having + // reserved two positions. A second event numbered off the log as this + // world sees it would take the slot the next start in the batch claimed, + // costing every start after the first its claim. + const runId = await newSlotRun(); + const claims = Array.from({ length: 10 }, (_, index) => + slotEventId(FIRST_SLOT + 2 * (index + 1)) + ); + const started = await Promise.all( + claims.map((eventId, index) => + events.create( + runId, + { + eventType: 'step_started', + specVersion: SPEC_VERSION_SLOT_IDENTITY, + correlationId: `step_${index}`, + eventData: { stepName: 'a-step', input: new Uint8Array() }, + }, + { eventId } + ) + ) + ); + expect(started.map((result) => result.event?.eventId)).toEqual(claims); + expect(ascending(await slotsOf(runId))).toEqual( + denseFrom(2 * claims.length + 1) + ); + }); + + test('rejects a claim that leaves no room for the second event', async () => { + // The run's own run_created holds the first slot, so a claim of the second + // means the caller reserved one position for a write that publishes two. + const runId = await newSlotRun(); + await expect( + events.create( + runId, + { + eventType: 'step_started', + specVersion: SPEC_VERSION_SLOT_IDENTITY, + correlationId: 'step_a', + eventData: { stepName: 'a-step', input: new Uint8Array() }, + }, + { eventId: slotEventId(FIRST_SLOT + 1) } + ) + ).rejects.toThrow(/leaves no slot below it/); }); test('numbers events of runs it never created', async () => { From d099af513d822b8c6877693b9cf5a42a769e9237 Mon Sep 17 00:00:00 2001 From: Peter Wielander Date: Fri, 31 Jul 2026 01:00:48 -0700 Subject: [PATCH 17/35] fix(core): floor a turbo run's slot claims above its own positions MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Turbo skips the initial event-log load and replays against an empty snapshot, so the log a claim is numbered from shows neither `run_created` nor the `run_started` still in flight. The floor was only raised from the backgrounded `run_started` response, which is too late to matter: a suspension reserves its whole batch of positions synchronously, so the first batch numbered from an empty log claims the two positions the run already holds, both ops lose their claims, and after the bounded reclaim retries one of them is dropped for good — the run then waits forever on an event that will never be written. Both positions are certain the moment turbo engages, so seed the floor there instead. Co-Authored-By: Claude Opus 5 --- packages/core/src/runtime.test.ts | 44 ++++++++++++++++++++++++++++--- packages/core/src/runtime.ts | 16 +++++++++++ 2 files changed, 56 insertions(+), 4 deletions(-) diff --git a/packages/core/src/runtime.test.ts b/packages/core/src/runtime.test.ts index aafec00075..3a42435b01 100644 --- a/packages/core/src/runtime.test.ts +++ b/packages/core/src/runtime.test.ts @@ -6,7 +6,10 @@ import { } from '@workflow/errors'; import { type Event, + FIRST_SLOT, SPEC_VERSION_CURRENT, + SPEC_VERSION_SLOT_IDENTITY, + slotFromId, type WorkflowRun, } from '@workflow/world'; import { ulid } from 'ulid'; @@ -1472,12 +1475,15 @@ describe('workflowEntrypoint turbo mode', () => { return r; }${xform('workflow')}`; - async function makeRunInput(runId: string) { + async function makeRunInput( + runId: string, + specVersion = SPEC_VERSION_CURRENT + ) { return { input: await dehydrateWorkflowArguments([], runId, undefined, []), deploymentId: 'test-deployment', workflowName: 'workflow', - specVersion: SPEC_VERSION_CURRENT, + specVersion, executionContext: {}, }; } @@ -1493,8 +1499,10 @@ describe('workflowEntrypoint turbo mode', () => { attempt: number; source: string; runStartedGate?: Promise; + specVersion?: typeof SPEC_VERSION_CURRENT; }) { const { runId, attempt, source } = opts; + const specVersion = opts.specVersion ?? SPEC_VERSION_CURRENT; const order = turboOrder; const durable: Event[] = []; let seq = 0; @@ -1513,6 +1521,7 @@ describe('workflowEntrypoint turbo mode', () => { runId, workflowName: 'workflow', status: 'running', + specVersion, input: await dehydrateWorkflowArguments([], runId, undefined, []), createdAt: new Date('2024-01-01T00:00:00.000Z'), updatedAt: new Date('2024-01-01T00:00:00.000Z'), @@ -1558,7 +1567,7 @@ describe('workflowEntrypoint turbo mode', () => { }); setWorld({ - specVersion: SPEC_VERSION_CURRENT, + specVersion, createQueueHandler: vi.fn( (_p: string, handler: (m: unknown, md: unknown) => Promise) => async () => { @@ -1566,7 +1575,7 @@ describe('workflowEntrypoint turbo mode', () => { { runId, requestedAt: new Date('2024-01-01T00:00:00.000Z'), - runInput: await makeRunInput(runId), + runInput: await makeRunInput(runId, specVersion), }, { requestId: 'req_turbo', @@ -1652,6 +1661,33 @@ describe('workflowEntrypoint turbo mode', () => { ); }); + it('claims slots above the run own positions on a first delivery', async () => { + // Turbo replays against an empty snapshot, so the log the claims are + // numbered from cannot show `run_created` or the in-flight `run_started`. + // Both positions are nonetheless taken, and the mocked `run_started` + // response reports no event — the same shape as a World that skips the + // preload — so nothing but the floor seeded at turbo entry keeps the first + // batch of claims off them. + const { handlerPromise, eventsCreate } = await driveTurbo({ + runId: 'wrun_turbo_slots', + attempt: 1, + source: stepAndSleepWorkflow, + specVersion: SPEC_VERSION_SLOT_IDENTITY, + }); + + const res = await handlerPromise; + expect(res.status).toBe(204); + + const claimed = eventsCreate.mock.calls + .map((c) => (c[2] as { eventId?: unknown } | undefined)?.eventId) + .filter((id): id is string => typeof id === 'string'); + // The sleep's `wait_created` is claimed, so there is something to assert on. + expect(claimed.length).toBeGreaterThan(0); + for (const eventId of claimed) { + expect(slotFromId(eventId)).toBeGreaterThan(FIRST_SLOT + 1); + } + }); + it('does not turbo when WORKFLOW_TURBO=0 (parity with the awaited path)', async () => { process.env.WORKFLOW_TURBO = '0'; const { handlerPromise, order } = await driveTurbo({ diff --git a/packages/core/src/runtime.ts b/packages/core/src/runtime.ts index 300c2c465c..7e8bb8c915 100644 --- a/packages/core/src/runtime.ts +++ b/packages/core/src/runtime.ts @@ -17,6 +17,7 @@ import { } from '@workflow/utils/parse-name'; import { type Event, + FIRST_SLOT, getQueueTopicPrefix, isLegacySpecVersion, ROOT_RUN_ID_ATTRIBUTE, @@ -24,6 +25,7 @@ import { SPEC_VERSION_CURRENT, SPEC_VERSION_SUPPORTS_COMPRESSION, slotFromId, + usesSlotIdentity, WorkflowInvokePayloadSchema, type WorkflowRun, type World, @@ -1056,6 +1058,20 @@ export function workflowEntrypoint( // intentionally truthy here — do not change the load // branches' `if (preloadedEvents)` checks to test length. preloadedEvents = []; + // A slot-numbered run's first two positions are the run's + // own: `run_created` from start(), then the `run_started` + // in flight above. Both are certain before any write of + // this invocation, and turbo replays against the empty + // snapshot skipped just above — so seed the floor with + // them here rather than waiting for the backgrounded + // response to report it. Waiting loses the race: a + // suspension reserves its whole batch of positions + // synchronously, so a batch that starts numbering from an + // empty log claims the two the run already holds and the + // ops holding them lose their claims. + if (usesSlotIdentity(runInput.specVersion)) { + knownSlotFloor = FIRST_SLOT + 1; + } const now = new Date(); workflowRun = { runId, From 992da0f05a6019fa78681a2ca72e2af07c1e7931 Mon Sep 17 00:00:00 2001 From: Peter Wielander Date: Fri, 31 Jul 2026 03:17:26 -0700 Subject: [PATCH 18/35] fix(world-local,world-postgres): undo an unpublished write's duplicate-suppression claims A create materializes its entity before it publishes the event, so a claimed slot that loses its position left the entity and its `.created` lock behind. The caller re-proposes the same op one slot higher and trips its own orphan, receiving a duplicate-entity error the runtime reads as "my write already landed" - the wait is never created and the run re-invokes forever. world-local now unwinds those claims when a create ends without publishing, and a synchronous claim is held even before the run has a slot book, so a concurrent allocation in the same process cannot hand the position away. world-postgres adopts an orphaned wait row the way it already adopts a hook. --- packages/world-local/src/fs.ts | 5 ++ .../world-local/src/storage/events-storage.ts | 45 ++++++++++++++--- .../src/storage/slot-identity.test.ts | 48 +++++++++++++++++++ .../world-local/src/storage/slots.test.ts | 33 +++++++++++++ packages/world-local/src/storage/slots.ts | 46 +++++++++++++++--- packages/world-postgres/src/storage.ts | 46 +++++++++++++++--- 6 files changed, 203 insertions(+), 20 deletions(-) diff --git a/packages/world-local/src/fs.ts b/packages/world-local/src/fs.ts index 8ae765d0f0..5827f36896 100644 --- a/packages/world-local/src/fs.ts +++ b/packages/world-local/src/fs.ts @@ -461,6 +461,11 @@ export async function deleteJSON(filePath: string): Promise { await fs.unlink(filePath); } catch (error) { if ((error as any).code !== 'ENOENT') throw error; + } finally { + // The cache stands in for an `fs.access` on the write path, so a path that + // no longer exists may not stay in it: a later create-if-absent write of the + // same path would be rejected as a duplicate of a file that is gone. + createdFilesCache.delete(filePath); } } diff --git a/packages/world-local/src/storage/events-storage.ts b/packages/world-local/src/storage/events-storage.ts index 558d5f7d03..ba4180c405 100644 --- a/packages/world-local/src/storage/events-storage.ts +++ b/packages/world-local/src/storage/events-storage.ts @@ -725,6 +725,15 @@ export function createEventsStorage( // be filled, and a log with a hole can no longer prove it is complete. const reserved = new Set(); let reservedRunId: string | undefined; + /** + * Undo actions for the duplicate-suppression claims a create takes before + * its event exists, newest last. Run only when the create ends without + * publishing anything: the answer to a lost position is to propose the + * same operation one position higher, and a claim left behind is what + * would reject that retry as a duplicate of a write that never landed. + */ + const abandonedClaims: Array<() => Promise> = []; + let eventCommitted = false; /** * Hands the slots of a create that never published back to the allocator, * so an abandoned reservation below a sibling's published slot does not @@ -741,6 +750,14 @@ export function createEventsStorage( slots.release(reservedRunId, slot); } } + if (!eventCommitted) { + for (const undo of abandonedClaims.reverse()) { + // Best effort: the throw the caller sees is the one that matters, + // and a claim that outlives its create is a duplicate suppressed + // for a write that is not coming back. + await undo().catch(() => {}); + } + } throw error; } } @@ -1657,6 +1674,7 @@ export function createEventsStorage( `Step "${data.correlationId}" already created` ); } + abandonedClaims.push(() => fs.unlink(stepCreatedLockPath)); const stepData = data.eventData as { stepName: string; input: any; @@ -1678,10 +1696,14 @@ export function createEventsStorage( specVersion: effectiveSpecVersion, }; const stepCompositeKey = `${effectiveRunId}-${data.correlationId}`; - await writeJSON( - taggedPath(basedir, 'steps', stepCompositeKey, tag), - step + const stepEntityPath = taggedPath( + basedir, + 'steps', + stepCompositeKey, + tag ); + await writeJSON(stepEntityPath, step); + abandonedClaims.push(() => deleteJSON(stepEntityPath)); } else if (data.eventType === 'step_started') { // step_started: Increments attempt, sets status to 'running' // Sets startedAt only on the first start (not updated on retries) @@ -2391,6 +2413,7 @@ export function createEventsStorage( `Wait "${data.correlationId}" already exists` ); } + abandonedClaims.push(() => fs.unlink(waitCreatedLockPath)); const waitData = data.eventData as { resumeAt?: Date; }; @@ -2404,10 +2427,14 @@ export function createEventsStorage( updatedAt: now, specVersion: effectiveSpecVersion, }; - await writeJSON( - taggedPath(basedir, 'waits', waitCompositeKey, tag), - wait + const waitEntityPath = taggedPath( + basedir, + 'waits', + waitCompositeKey, + tag ); + await writeJSON(waitEntityPath, wait); + abandonedClaims.push(() => deleteJSON(waitEntityPath)); } else if (data.eventType === 'wait_completed') { // wait_completed: Transitions wait to 'completed', rejects duplicates. // Uses writeExclusive on a lock file to atomically prevent concurrent @@ -2713,7 +2740,11 @@ export function createEventsStorage( } // The event is now committed; cache it so an immediate sequential - // replay can serve it without rereading from disk. + // replay can serve it without rereading from disk. Nothing this create + // claimed may be undone from here on: readers can see the event, so the + // entity it describes has to keep existing even if a later step of this + // call fails. + eventCommitted = true; rememberStoredEvent(event, eventPath, serializedEvent); slots.observe(effectiveRunId, eventId); if (companionSlot !== undefined) { diff --git a/packages/world-local/src/storage/slot-identity.test.ts b/packages/world-local/src/storage/slot-identity.test.ts index 885126e5fa..e364584989 100644 --- a/packages/world-local/src/storage/slot-identity.test.ts +++ b/packages/world-local/src/storage/slot-identity.test.ts @@ -77,6 +77,27 @@ async function createStep( return result.event.eventId; } +async function createWait( + runId: string, + waitId: string, + eventId?: string +): Promise { + const result = await storage.events.create( + runId, + { + eventType: 'wait_created', + specVersion: SPEC_VERSION_SLOT_IDENTITY, + correlationId: waitId, + eventData: { resumeAt: new Date('2030-01-01T00:00:00.000Z') }, + }, + eventId === undefined ? undefined : { eventId } + ); + if (!result.event) { + throw new Error('Expected an event'); + } + return result.event.eventId; +} + describe('numbering', () => { it('puts run_created in the first slot', async () => { const runId = await newSlotRun(); @@ -345,6 +366,33 @@ describe('conflict', () => { await expect(slotsOf(runId)).resolves.toEqual([1, 2]); }); + it('lets a lost claim re-propose an entity it had already materialized', async () => { + // A claim only reaches its exclusive write after the entity it describes + // exists, so a claim that loses leaves that entity behind. The caller's + // whole answer to a conflict is to merge, replay and propose the same + // operation one position higher — which it cannot do if its own leftover + // entity is what rejects the retry. + const runId = await newSlotRun(); + // Seed this instance's book, then let another instance take the position + // the book will hand out next. The claim below passes the book's + // "is it written?" check because the book has not seen that write. + await createStep(runId, 'step_seed'); + const other = createStorage(testDir); + await other.events.create(runId, { + eventType: 'step_created', + specVersion: SPEC_VERSION_SLOT_IDENTITY, + correlationId: 'step_out_of_band', + eventData: { stepName: 'b-step', input: new Uint8Array() }, + }); + + await expect(createWait(runId, 'wait_a', slotEventId(3))).rejects.toThrow( + SlotConflictError + ); + const eventId = await createWait(runId, 'wait_a', slotEventId(4)); + expect(eventId).toBe(slotEventId(4)); + await expect(slotsOf(runId)).resolves.toEqual([1, 2, 3, 4]); + }); + it('reallocates around another instance holding the slot it picked', async () => { // Neither writer holds a log, so neither has anything to reconcile: the // loser takes the next free position instead of surfacing a conflict its diff --git a/packages/world-local/src/storage/slots.test.ts b/packages/world-local/src/storage/slots.test.ts index 42c962f128..0e60b2ff74 100644 --- a/packages/world-local/src/storage/slots.test.ts +++ b/packages/world-local/src/storage/slots.test.ts @@ -203,6 +203,39 @@ describe('isWritten', () => { }); }); +describe('claim', () => { + it('holds a slot claimed before anything allocated for the run', async () => { + // A claim is synchronous and the first allocation's log scan is not, so a + // claim that only registered against an existing book would be invisible to + // the very allocation it races — and a single-process app would hand the + // caller's own position away. + await writeEvents(1); + const book = createSlotBook(basedir); + book.claim(RUN_ID, 2); + await expect(book.reserve(RUN_ID)).resolves.toBe(3); + }); + + it('frees the slot again once the claim resolves', async () => { + await writeEvents(1); + const book = createSlotBook(basedir); + book.claim(RUN_ID, 2); + book.release(RUN_ID, 2); + // Nothing is allocating for the run yet, so the book the next caller seeds + // has to start from the log alone. + await expect(book.reserve(RUN_ID)).resolves.toBe(2); + }); + + it('keeps holding a claim across a forget', async () => { + // `forget` follows a lost publish: the book is behind another writer, but + // the claims other writes in this instance still hold are not. + await writeEvents(1); + const book = createSlotBook(basedir); + book.claim(RUN_ID, 2); + book.forget(RUN_ID); + await expect(book.reserve(RUN_ID)).resolves.toBe(3); + }); +}); + describe('observe', () => { it('never hands out a slot claimed by the client', async () => { const book = createSlotBook(basedir); diff --git a/packages/world-local/src/storage/slots.ts b/packages/world-local/src/storage/slots.ts index 632e31c1ee..58275f3d57 100644 --- a/packages/world-local/src/storage/slots.ts +++ b/packages/world-local/src/storage/slots.ts @@ -118,6 +118,14 @@ export function createSlotBook(basedir: string, tag?: string): SlotBook { const books = new Map(); /** runId → in-flight seed scan, so concurrent first callers share one scan. */ const seeds = new Map>(); + /** + * runId → slots claimed while the run had no book yet, so the book the next + * allocation seeds starts out holding them. A claim is synchronous and a seed + * scan is not: without this, the first allocation of a run would read the log + * from disk and hand out a position a caller in this very instance had already + * claimed — the case that makes a claim lose in a single-process app. + */ + const claims = new Map>(); async function readMode(runId: string): Promise { const run = await readJSONWithFallback( @@ -141,7 +149,7 @@ export function createSlotBook(basedir: string, tag?: string): SlotBook { } const book: RunSlots = { written, - outstanding: new Set(), + outstanding: new Set(claims.get(runId)), searchFrom: FIRST_SLOT, }; books.set(runId, book); @@ -162,6 +170,21 @@ export function createSlotBook(basedir: string, tag?: string): SlotBook { return pending; } + /** + * Drops a claim once its publish resolved, either way: a claim left behind + * would be handed to no one and become a hole in a log seeded later. + */ + function forgetClaim(runId: string, slot: number): void { + const claimed = claims.get(runId); + if (!claimed) { + return; + } + claimed.delete(slot); + if (claimed.size === 0) { + claims.delete(runId); + } + } + function take(book: RunSlots, minSlot: number): number { let slot = Math.max(book.searchFrom, minSlot); while (book.written.has(slot) || book.outstanding.has(slot)) { @@ -201,8 +224,12 @@ export function createSlotBook(basedir: string, tag?: string): SlotBook { }, claim(runId, slot) { - // No book means nothing is allocating for this run in this instance yet, - // and the seed scan that starts one reads the claim off disk if it landed. + const claimed = claims.get(runId); + if (claimed) { + claimed.add(slot); + } else { + claims.set(runId, new Set([slot])); + } books.get(runId)?.outstanding.add(slot); }, @@ -212,6 +239,7 @@ export function createSlotBook(basedir: string, tag?: string): SlotBook { }, release(runId, slot) { + forgetClaim(runId, slot); const book = books.get(runId); if (!book) { return; @@ -223,16 +251,17 @@ export function createSlotBook(basedir: string, tag?: string): SlotBook { }, observe(runId, eventId) { + const slot = slotFromId(eventId); + if (slot === undefined) { + return; + } + forgetClaim(runId, slot); const book = books.get(runId); if (!book) { // Nothing to keep consistent: the slot is on disk by the time this is // called, so the eventual seed scan picks it up. return; } - const slot = slotFromId(eventId); - if (slot === undefined) { - return; - } book.written.add(slot); book.outstanding.delete(slot); }, @@ -258,11 +287,14 @@ export function createSlotBook(basedir: string, tag?: string): SlotBook { forget(runId) { modes.delete(runId); books.delete(runId); + // Claims outlive the book on purpose: they belong to writes still in + // flight, and the book a later allocation seeds has to hold them back. }, clear() { modes.clear(); books.clear(); + claims.clear(); }, }; } diff --git a/packages/world-postgres/src/storage.ts b/packages/world-postgres/src/storage.ts index ab752f7de5..b62a17ec6f 100644 --- a/packages/world-postgres/src/storage.ts +++ b/packages/world-postgres/src/storage.ts @@ -456,7 +456,7 @@ export function createEventsStorage(drizzle: Drizzle): Storage['events'] { // hook row left behind by a process / database interruption between // the hook INSERT and the events INSERT below (see the recovery // logic in the hook_created branch). - const getHookCreatedEvent = drizzle + const getCorrelatedEvent = drizzle .select({ eventId: events.eventId }) .from(events) .where( @@ -467,7 +467,7 @@ export function createEventsStorage(drizzle: Drizzle): Storage['events'] { ) ) .limit(1) - .prepare('events_get_hook_created_for_run_correlation'); + .prepare('events_get_correlated_event'); const getWaitForValidation = drizzle .select({ @@ -1691,7 +1691,7 @@ export function createEventsStorage(drizzle: Drizzle): Storage['events'] { existingHook.runId === effectiveRunId && existingHook.hookId === data.correlationId ) { - const [existingEvent] = await getHookCreatedEvent.execute({ + const [existingEvent] = await getCorrelatedEvent.execute({ runId: effectiveRunId, correlationId: data.correlationId, eventType: 'hook_created', @@ -1892,9 +1892,43 @@ export function createEventsStorage(drizzle: Drizzle): Storage['events'] { specVersion: waitValue.specVersion ?? undefined, }; } else { - throw new EntityConflictError( - `Wait "${data.correlationId}" already exists` - ); + // The wait row exists but this call did not write it. Which of the two + // reasons it is decides everything, and only the event log knows: + // - the `wait_created` event exists → a real duplicate, so throw and + // let the runtime's concurrent-replay catch path swallow it. + // - it does not → an orphaned row from an attempt that materialized + // the wait and then lost its event write (a crash, or a slot + // claimed by someone else). The caller re-proposing the same + // operation one position higher is exactly what has to succeed + // here, so adopt the row and complete the partial write. Mirrors + // hook_created's handling of the same window. + const [existingEvent] = await getCorrelatedEvent.execute({ + runId: effectiveRunId, + correlationId: data.correlationId, + eventType: 'wait_created', + }); + if (existingEvent) { + throw new EntityConflictError( + `Wait "${data.correlationId}" already exists` + ); + } + const [orphan] = await drizzle + .select() + .from(Schema.waits) + .where(eq(Schema.waits.waitId, waitId)) + .limit(1); + if (orphan) { + wait = { + waitId: orphan.waitId, + runId: orphan.runId, + status: orphan.status, + resumeAt: orphan.resumeAt ?? undefined, + completedAt: orphan.completedAt ?? undefined, + createdAt: orphan.createdAt, + updatedAt: orphan.updatedAt, + specVersion: orphan.specVersion ?? undefined, + }; + } } } From e4076751e123d11db7ef30ed91e0e784b2c3dbe8 Mon Sep 17 00:00:00 2001 From: Peter Wielander Date: Fri, 31 Jul 2026 03:17:26 -0700 Subject: [PATCH 19/35] fix(world-local,world-postgres): undo an unpublished write's duplicate-suppression claims A create materializes its entity before it publishes the event, so a claimed slot that loses its position left the entity and its `.created` lock behind. The caller re-proposes the same op one slot higher and trips its own orphan, receiving a duplicate-entity error the runtime reads as "my write already landed" - the wait is never created and the run re-invokes forever. world-local now unwinds those claims when a create ends without publishing, and a synchronous claim is held even before the run has a slot book, so a concurrent allocation in the same process cannot hand the position away. world-postgres adopts an orphaned wait row the way it already adopts a hook. --- packages/world-local/src/fs.ts | 5 ++ .../world-local/src/storage/events-storage.ts | 45 ++++++++++++++--- .../src/storage/slot-identity.test.ts | 48 +++++++++++++++++++ .../world-local/src/storage/slots.test.ts | 33 +++++++++++++ packages/world-local/src/storage/slots.ts | 46 +++++++++++++++--- packages/world-postgres/src/storage.ts | 46 +++++++++++++++--- 6 files changed, 203 insertions(+), 20 deletions(-) diff --git a/packages/world-local/src/fs.ts b/packages/world-local/src/fs.ts index 70178b344c..aa3db9ee1e 100644 --- a/packages/world-local/src/fs.ts +++ b/packages/world-local/src/fs.ts @@ -466,6 +466,11 @@ export async function deleteJSON(filePath: string): Promise { await withWindowsRetry(() => fs.unlink(filePath)); } catch (error) { if ((error as any).code !== 'ENOENT') throw error; + } finally { + // The cache stands in for an `fs.access` on the write path, so a path that + // no longer exists may not stay in it: a later create-if-absent write of the + // same path would be rejected as a duplicate of a file that is gone. + createdFilesCache.delete(filePath); } } diff --git a/packages/world-local/src/storage/events-storage.ts b/packages/world-local/src/storage/events-storage.ts index 558d5f7d03..ba4180c405 100644 --- a/packages/world-local/src/storage/events-storage.ts +++ b/packages/world-local/src/storage/events-storage.ts @@ -725,6 +725,15 @@ export function createEventsStorage( // be filled, and a log with a hole can no longer prove it is complete. const reserved = new Set(); let reservedRunId: string | undefined; + /** + * Undo actions for the duplicate-suppression claims a create takes before + * its event exists, newest last. Run only when the create ends without + * publishing anything: the answer to a lost position is to propose the + * same operation one position higher, and a claim left behind is what + * would reject that retry as a duplicate of a write that never landed. + */ + const abandonedClaims: Array<() => Promise> = []; + let eventCommitted = false; /** * Hands the slots of a create that never published back to the allocator, * so an abandoned reservation below a sibling's published slot does not @@ -741,6 +750,14 @@ export function createEventsStorage( slots.release(reservedRunId, slot); } } + if (!eventCommitted) { + for (const undo of abandonedClaims.reverse()) { + // Best effort: the throw the caller sees is the one that matters, + // and a claim that outlives its create is a duplicate suppressed + // for a write that is not coming back. + await undo().catch(() => {}); + } + } throw error; } } @@ -1657,6 +1674,7 @@ export function createEventsStorage( `Step "${data.correlationId}" already created` ); } + abandonedClaims.push(() => fs.unlink(stepCreatedLockPath)); const stepData = data.eventData as { stepName: string; input: any; @@ -1678,10 +1696,14 @@ export function createEventsStorage( specVersion: effectiveSpecVersion, }; const stepCompositeKey = `${effectiveRunId}-${data.correlationId}`; - await writeJSON( - taggedPath(basedir, 'steps', stepCompositeKey, tag), - step + const stepEntityPath = taggedPath( + basedir, + 'steps', + stepCompositeKey, + tag ); + await writeJSON(stepEntityPath, step); + abandonedClaims.push(() => deleteJSON(stepEntityPath)); } else if (data.eventType === 'step_started') { // step_started: Increments attempt, sets status to 'running' // Sets startedAt only on the first start (not updated on retries) @@ -2391,6 +2413,7 @@ export function createEventsStorage( `Wait "${data.correlationId}" already exists` ); } + abandonedClaims.push(() => fs.unlink(waitCreatedLockPath)); const waitData = data.eventData as { resumeAt?: Date; }; @@ -2404,10 +2427,14 @@ export function createEventsStorage( updatedAt: now, specVersion: effectiveSpecVersion, }; - await writeJSON( - taggedPath(basedir, 'waits', waitCompositeKey, tag), - wait + const waitEntityPath = taggedPath( + basedir, + 'waits', + waitCompositeKey, + tag ); + await writeJSON(waitEntityPath, wait); + abandonedClaims.push(() => deleteJSON(waitEntityPath)); } else if (data.eventType === 'wait_completed') { // wait_completed: Transitions wait to 'completed', rejects duplicates. // Uses writeExclusive on a lock file to atomically prevent concurrent @@ -2713,7 +2740,11 @@ export function createEventsStorage( } // The event is now committed; cache it so an immediate sequential - // replay can serve it without rereading from disk. + // replay can serve it without rereading from disk. Nothing this create + // claimed may be undone from here on: readers can see the event, so the + // entity it describes has to keep existing even if a later step of this + // call fails. + eventCommitted = true; rememberStoredEvent(event, eventPath, serializedEvent); slots.observe(effectiveRunId, eventId); if (companionSlot !== undefined) { diff --git a/packages/world-local/src/storage/slot-identity.test.ts b/packages/world-local/src/storage/slot-identity.test.ts index 885126e5fa..e364584989 100644 --- a/packages/world-local/src/storage/slot-identity.test.ts +++ b/packages/world-local/src/storage/slot-identity.test.ts @@ -77,6 +77,27 @@ async function createStep( return result.event.eventId; } +async function createWait( + runId: string, + waitId: string, + eventId?: string +): Promise { + const result = await storage.events.create( + runId, + { + eventType: 'wait_created', + specVersion: SPEC_VERSION_SLOT_IDENTITY, + correlationId: waitId, + eventData: { resumeAt: new Date('2030-01-01T00:00:00.000Z') }, + }, + eventId === undefined ? undefined : { eventId } + ); + if (!result.event) { + throw new Error('Expected an event'); + } + return result.event.eventId; +} + describe('numbering', () => { it('puts run_created in the first slot', async () => { const runId = await newSlotRun(); @@ -345,6 +366,33 @@ describe('conflict', () => { await expect(slotsOf(runId)).resolves.toEqual([1, 2]); }); + it('lets a lost claim re-propose an entity it had already materialized', async () => { + // A claim only reaches its exclusive write after the entity it describes + // exists, so a claim that loses leaves that entity behind. The caller's + // whole answer to a conflict is to merge, replay and propose the same + // operation one position higher — which it cannot do if its own leftover + // entity is what rejects the retry. + const runId = await newSlotRun(); + // Seed this instance's book, then let another instance take the position + // the book will hand out next. The claim below passes the book's + // "is it written?" check because the book has not seen that write. + await createStep(runId, 'step_seed'); + const other = createStorage(testDir); + await other.events.create(runId, { + eventType: 'step_created', + specVersion: SPEC_VERSION_SLOT_IDENTITY, + correlationId: 'step_out_of_band', + eventData: { stepName: 'b-step', input: new Uint8Array() }, + }); + + await expect(createWait(runId, 'wait_a', slotEventId(3))).rejects.toThrow( + SlotConflictError + ); + const eventId = await createWait(runId, 'wait_a', slotEventId(4)); + expect(eventId).toBe(slotEventId(4)); + await expect(slotsOf(runId)).resolves.toEqual([1, 2, 3, 4]); + }); + it('reallocates around another instance holding the slot it picked', async () => { // Neither writer holds a log, so neither has anything to reconcile: the // loser takes the next free position instead of surfacing a conflict its diff --git a/packages/world-local/src/storage/slots.test.ts b/packages/world-local/src/storage/slots.test.ts index 42c962f128..0e60b2ff74 100644 --- a/packages/world-local/src/storage/slots.test.ts +++ b/packages/world-local/src/storage/slots.test.ts @@ -203,6 +203,39 @@ describe('isWritten', () => { }); }); +describe('claim', () => { + it('holds a slot claimed before anything allocated for the run', async () => { + // A claim is synchronous and the first allocation's log scan is not, so a + // claim that only registered against an existing book would be invisible to + // the very allocation it races — and a single-process app would hand the + // caller's own position away. + await writeEvents(1); + const book = createSlotBook(basedir); + book.claim(RUN_ID, 2); + await expect(book.reserve(RUN_ID)).resolves.toBe(3); + }); + + it('frees the slot again once the claim resolves', async () => { + await writeEvents(1); + const book = createSlotBook(basedir); + book.claim(RUN_ID, 2); + book.release(RUN_ID, 2); + // Nothing is allocating for the run yet, so the book the next caller seeds + // has to start from the log alone. + await expect(book.reserve(RUN_ID)).resolves.toBe(2); + }); + + it('keeps holding a claim across a forget', async () => { + // `forget` follows a lost publish: the book is behind another writer, but + // the claims other writes in this instance still hold are not. + await writeEvents(1); + const book = createSlotBook(basedir); + book.claim(RUN_ID, 2); + book.forget(RUN_ID); + await expect(book.reserve(RUN_ID)).resolves.toBe(3); + }); +}); + describe('observe', () => { it('never hands out a slot claimed by the client', async () => { const book = createSlotBook(basedir); diff --git a/packages/world-local/src/storage/slots.ts b/packages/world-local/src/storage/slots.ts index 632e31c1ee..58275f3d57 100644 --- a/packages/world-local/src/storage/slots.ts +++ b/packages/world-local/src/storage/slots.ts @@ -118,6 +118,14 @@ export function createSlotBook(basedir: string, tag?: string): SlotBook { const books = new Map(); /** runId → in-flight seed scan, so concurrent first callers share one scan. */ const seeds = new Map>(); + /** + * runId → slots claimed while the run had no book yet, so the book the next + * allocation seeds starts out holding them. A claim is synchronous and a seed + * scan is not: without this, the first allocation of a run would read the log + * from disk and hand out a position a caller in this very instance had already + * claimed — the case that makes a claim lose in a single-process app. + */ + const claims = new Map>(); async function readMode(runId: string): Promise { const run = await readJSONWithFallback( @@ -141,7 +149,7 @@ export function createSlotBook(basedir: string, tag?: string): SlotBook { } const book: RunSlots = { written, - outstanding: new Set(), + outstanding: new Set(claims.get(runId)), searchFrom: FIRST_SLOT, }; books.set(runId, book); @@ -162,6 +170,21 @@ export function createSlotBook(basedir: string, tag?: string): SlotBook { return pending; } + /** + * Drops a claim once its publish resolved, either way: a claim left behind + * would be handed to no one and become a hole in a log seeded later. + */ + function forgetClaim(runId: string, slot: number): void { + const claimed = claims.get(runId); + if (!claimed) { + return; + } + claimed.delete(slot); + if (claimed.size === 0) { + claims.delete(runId); + } + } + function take(book: RunSlots, minSlot: number): number { let slot = Math.max(book.searchFrom, minSlot); while (book.written.has(slot) || book.outstanding.has(slot)) { @@ -201,8 +224,12 @@ export function createSlotBook(basedir: string, tag?: string): SlotBook { }, claim(runId, slot) { - // No book means nothing is allocating for this run in this instance yet, - // and the seed scan that starts one reads the claim off disk if it landed. + const claimed = claims.get(runId); + if (claimed) { + claimed.add(slot); + } else { + claims.set(runId, new Set([slot])); + } books.get(runId)?.outstanding.add(slot); }, @@ -212,6 +239,7 @@ export function createSlotBook(basedir: string, tag?: string): SlotBook { }, release(runId, slot) { + forgetClaim(runId, slot); const book = books.get(runId); if (!book) { return; @@ -223,16 +251,17 @@ export function createSlotBook(basedir: string, tag?: string): SlotBook { }, observe(runId, eventId) { + const slot = slotFromId(eventId); + if (slot === undefined) { + return; + } + forgetClaim(runId, slot); const book = books.get(runId); if (!book) { // Nothing to keep consistent: the slot is on disk by the time this is // called, so the eventual seed scan picks it up. return; } - const slot = slotFromId(eventId); - if (slot === undefined) { - return; - } book.written.add(slot); book.outstanding.delete(slot); }, @@ -258,11 +287,14 @@ export function createSlotBook(basedir: string, tag?: string): SlotBook { forget(runId) { modes.delete(runId); books.delete(runId); + // Claims outlive the book on purpose: they belong to writes still in + // flight, and the book a later allocation seeds has to hold them back. }, clear() { modes.clear(); books.clear(); + claims.clear(); }, }; } diff --git a/packages/world-postgres/src/storage.ts b/packages/world-postgres/src/storage.ts index ab752f7de5..b62a17ec6f 100644 --- a/packages/world-postgres/src/storage.ts +++ b/packages/world-postgres/src/storage.ts @@ -456,7 +456,7 @@ export function createEventsStorage(drizzle: Drizzle): Storage['events'] { // hook row left behind by a process / database interruption between // the hook INSERT and the events INSERT below (see the recovery // logic in the hook_created branch). - const getHookCreatedEvent = drizzle + const getCorrelatedEvent = drizzle .select({ eventId: events.eventId }) .from(events) .where( @@ -467,7 +467,7 @@ export function createEventsStorage(drizzle: Drizzle): Storage['events'] { ) ) .limit(1) - .prepare('events_get_hook_created_for_run_correlation'); + .prepare('events_get_correlated_event'); const getWaitForValidation = drizzle .select({ @@ -1691,7 +1691,7 @@ export function createEventsStorage(drizzle: Drizzle): Storage['events'] { existingHook.runId === effectiveRunId && existingHook.hookId === data.correlationId ) { - const [existingEvent] = await getHookCreatedEvent.execute({ + const [existingEvent] = await getCorrelatedEvent.execute({ runId: effectiveRunId, correlationId: data.correlationId, eventType: 'hook_created', @@ -1892,9 +1892,43 @@ export function createEventsStorage(drizzle: Drizzle): Storage['events'] { specVersion: waitValue.specVersion ?? undefined, }; } else { - throw new EntityConflictError( - `Wait "${data.correlationId}" already exists` - ); + // The wait row exists but this call did not write it. Which of the two + // reasons it is decides everything, and only the event log knows: + // - the `wait_created` event exists → a real duplicate, so throw and + // let the runtime's concurrent-replay catch path swallow it. + // - it does not → an orphaned row from an attempt that materialized + // the wait and then lost its event write (a crash, or a slot + // claimed by someone else). The caller re-proposing the same + // operation one position higher is exactly what has to succeed + // here, so adopt the row and complete the partial write. Mirrors + // hook_created's handling of the same window. + const [existingEvent] = await getCorrelatedEvent.execute({ + runId: effectiveRunId, + correlationId: data.correlationId, + eventType: 'wait_created', + }); + if (existingEvent) { + throw new EntityConflictError( + `Wait "${data.correlationId}" already exists` + ); + } + const [orphan] = await drizzle + .select() + .from(Schema.waits) + .where(eq(Schema.waits.waitId, waitId)) + .limit(1); + if (orphan) { + wait = { + waitId: orphan.waitId, + runId: orphan.runId, + status: orphan.status, + resumeAt: orphan.resumeAt ?? undefined, + completedAt: orphan.completedAt ?? undefined, + createdAt: orphan.createdAt, + updatedAt: orphan.updatedAt, + specVersion: orphan.specVersion ?? undefined, + }; + } } } From 3cecc5290a0ba5414d1397cab9d5ef0537dd1537 Mon Sep 17 00:00:00 2001 From: Peter Wielander Date: Fri, 31 Jul 2026 03:36:45 -0700 Subject: [PATCH 20/35] fix(world-local): publish a deferred step_created on the same terms as its start The synthetic `step_created` a lazy step start materializes was written with a plain create-if-absent write, so losing its position surfaced as a duplicate- entity error - which the runtime reads as "another handler owns this step" and skips, leaving the step claimed but never run. It now takes the position the same way every other event does, and a loss is reported as a slot conflict the caller can merge and re-propose. --- .../world-local/src/storage/events-storage.ts | 60 ++++++++++++++----- .../src/storage/slot-identity.test.ts | 23 +++++++ 2 files changed, 67 insertions(+), 16 deletions(-) diff --git a/packages/world-local/src/storage/events-storage.ts b/packages/world-local/src/storage/events-storage.ts index ba4180c405..97376f5aab 100644 --- a/packages/world-local/src/storage/events-storage.ts +++ b/packages/world-local/src/storage/events-storage.ts @@ -733,6 +733,11 @@ export function createEventsStorage( * would reject that retry as a duplicate of a write that never landed. */ const abandonedClaims: Array<() => Promise> = []; + /** + * Whether any event of this create became reader-visible. Once one has, + * nothing the create claimed may be undone: the entity an event describes + * has to keep existing even if a later step of the same call fails. + */ let eventCommitted = false; /** * Hands the slots of a create that never published back to the allocator, @@ -1742,6 +1747,7 @@ export function createEventsStorage( `Step "${data.correlationId}" already created` ); } else { + abandonedClaims.push(() => fs.unlink(stepCreatedLockPath)); const createdStep: Step = { runId: effectiveRunId, stepId: data.correlationId, @@ -1757,15 +1763,14 @@ export function createEventsStorage( updatedAt: now, specVersion: effectiveSpecVersion, }; - await writeJSON( - taggedPath( - basedir, - 'steps', - `${effectiveRunId}-${data.correlationId}`, - tag - ), - createdStep + const lazyStepEntityPath = taggedPath( + basedir, + 'steps', + `${effectiveRunId}-${data.correlationId}`, + tag ); + await writeJSON(lazyStepEntityPath, createdStep); + abandonedClaims.push(() => deleteJSON(lazyStepEntityPath)); // Write the synthetic step_created event so replay observes it // (the client step consumer sets hasCreatedEvent only on a // step_created event). Its eventId is a second slot, or a fresh @@ -1799,15 +1804,38 @@ export function createEventsStorage( input: lazyData.input, }, }; - await writeJSON( - taggedPath( - basedir, - 'events', - `${effectiveRunId}-${stepCreatedEventId}`, - tag - ), - stepCreatedEvent + const stepCreatedEventPath = taggedPath( + basedir, + 'events', + `${effectiveRunId}-${stepCreatedEventId}`, + tag ); + if (slotMode) { + // The position decides this event as much as it decides the + // start it rides with, so it is published the same way: whoever + // links the file first owns the slot. A loss here is the + // caller's to resolve — it named this position — and the undo + // list above takes the step entity and its claim back out, so + // the re-proposal one position higher starts the step lazily + // again instead of tripping its own leftovers. + const published = await writeExclusive( + stepCreatedEventPath, + JSON.stringify(stepCreatedEvent, jsonReplacer, 2) + ); + if (!published) { + throw await slotConflict( + effectiveRunId, + stepCreatedEventId, + params + ); + } + } else { + await writeJSON(stepCreatedEventPath, stepCreatedEvent); + } + // Readers can see this event from here on, so the step entity it + // describes has to keep existing even if the start it rides with + // goes on to lose its own position. + eventCommitted = true; slots.observe(effectiveRunId, stepCreatedEventId); validatedStep = createdStep; stepCreatedLazily = true; diff --git a/packages/world-local/src/storage/slot-identity.test.ts b/packages/world-local/src/storage/slot-identity.test.ts index e364584989..c21e215afa 100644 --- a/packages/world-local/src/storage/slot-identity.test.ts +++ b/packages/world-local/src/storage/slot-identity.test.ts @@ -393,6 +393,29 @@ describe('conflict', () => { await expect(slotsOf(runId)).resolves.toEqual([1, 2, 3, 4]); }); + it('lets a lazy start lose the position of the event it defers', async () => { + // The deferred `step_created` is published on the same terms as the start + // itself, so it is the pair's first position that can be lost. The retry has + // to be able to start the step lazily all over again — its own claim file + // and step entity would otherwise answer for a write that never landed. + const runId = await newSlotRun(); + await createStep(runId, 'step_seed'); + const other = createStorage(testDir); + await other.events.create(runId, { + eventType: 'step_created', + specVersion: SPEC_VERSION_SLOT_IDENTITY, + correlationId: 'step_out_of_band', + eventData: { stepName: 'b-step', input: new Uint8Array() }, + }); + + await expect( + startStepLazily(runId, 'step_a', slotEventId(4)) + ).rejects.toThrow(SlotConflictError); + const eventId = await startStepLazily(runId, 'step_a', slotEventId(5)); + expect(eventId).toBe(slotEventId(5)); + await expect(slotsOf(runId)).resolves.toEqual([1, 2, 3, 4, 5]); + }); + it('reallocates around another instance holding the slot it picked', async () => { // Neither writer holds a log, so neither has anything to reconcile: the // loser takes the next free position instead of surfacing a conflict its From f73bbcddc25a2a3c8462c7331e71ff611c053ee1 Mon Sep 17 00:00:00 2001 From: Peter Wielander Date: Fri, 31 Jul 2026 03:36:45 -0700 Subject: [PATCH 21/35] fix(world-local): publish a deferred step_created on the same terms as its start The synthetic `step_created` a lazy step start materializes was written with a plain create-if-absent write, so losing its position surfaced as a duplicate- entity error - which the runtime reads as "another handler owns this step" and skips, leaving the step claimed but never run. It now takes the position the same way every other event does, and a loss is reported as a slot conflict the caller can merge and re-propose. --- .../world-local/src/storage/events-storage.ts | 60 ++++++++++++++----- .../src/storage/slot-identity.test.ts | 23 +++++++ 2 files changed, 67 insertions(+), 16 deletions(-) diff --git a/packages/world-local/src/storage/events-storage.ts b/packages/world-local/src/storage/events-storage.ts index ba4180c405..97376f5aab 100644 --- a/packages/world-local/src/storage/events-storage.ts +++ b/packages/world-local/src/storage/events-storage.ts @@ -733,6 +733,11 @@ export function createEventsStorage( * would reject that retry as a duplicate of a write that never landed. */ const abandonedClaims: Array<() => Promise> = []; + /** + * Whether any event of this create became reader-visible. Once one has, + * nothing the create claimed may be undone: the entity an event describes + * has to keep existing even if a later step of the same call fails. + */ let eventCommitted = false; /** * Hands the slots of a create that never published back to the allocator, @@ -1742,6 +1747,7 @@ export function createEventsStorage( `Step "${data.correlationId}" already created` ); } else { + abandonedClaims.push(() => fs.unlink(stepCreatedLockPath)); const createdStep: Step = { runId: effectiveRunId, stepId: data.correlationId, @@ -1757,15 +1763,14 @@ export function createEventsStorage( updatedAt: now, specVersion: effectiveSpecVersion, }; - await writeJSON( - taggedPath( - basedir, - 'steps', - `${effectiveRunId}-${data.correlationId}`, - tag - ), - createdStep + const lazyStepEntityPath = taggedPath( + basedir, + 'steps', + `${effectiveRunId}-${data.correlationId}`, + tag ); + await writeJSON(lazyStepEntityPath, createdStep); + abandonedClaims.push(() => deleteJSON(lazyStepEntityPath)); // Write the synthetic step_created event so replay observes it // (the client step consumer sets hasCreatedEvent only on a // step_created event). Its eventId is a second slot, or a fresh @@ -1799,15 +1804,38 @@ export function createEventsStorage( input: lazyData.input, }, }; - await writeJSON( - taggedPath( - basedir, - 'events', - `${effectiveRunId}-${stepCreatedEventId}`, - tag - ), - stepCreatedEvent + const stepCreatedEventPath = taggedPath( + basedir, + 'events', + `${effectiveRunId}-${stepCreatedEventId}`, + tag ); + if (slotMode) { + // The position decides this event as much as it decides the + // start it rides with, so it is published the same way: whoever + // links the file first owns the slot. A loss here is the + // caller's to resolve — it named this position — and the undo + // list above takes the step entity and its claim back out, so + // the re-proposal one position higher starts the step lazily + // again instead of tripping its own leftovers. + const published = await writeExclusive( + stepCreatedEventPath, + JSON.stringify(stepCreatedEvent, jsonReplacer, 2) + ); + if (!published) { + throw await slotConflict( + effectiveRunId, + stepCreatedEventId, + params + ); + } + } else { + await writeJSON(stepCreatedEventPath, stepCreatedEvent); + } + // Readers can see this event from here on, so the step entity it + // describes has to keep existing even if the start it rides with + // goes on to lose its own position. + eventCommitted = true; slots.observe(effectiveRunId, stepCreatedEventId); validatedStep = createdStep; stepCreatedLazily = true; diff --git a/packages/world-local/src/storage/slot-identity.test.ts b/packages/world-local/src/storage/slot-identity.test.ts index e364584989..c21e215afa 100644 --- a/packages/world-local/src/storage/slot-identity.test.ts +++ b/packages/world-local/src/storage/slot-identity.test.ts @@ -393,6 +393,29 @@ describe('conflict', () => { await expect(slotsOf(runId)).resolves.toEqual([1, 2, 3, 4]); }); + it('lets a lazy start lose the position of the event it defers', async () => { + // The deferred `step_created` is published on the same terms as the start + // itself, so it is the pair's first position that can be lost. The retry has + // to be able to start the step lazily all over again — its own claim file + // and step entity would otherwise answer for a write that never landed. + const runId = await newSlotRun(); + await createStep(runId, 'step_seed'); + const other = createStorage(testDir); + await other.events.create(runId, { + eventType: 'step_created', + specVersion: SPEC_VERSION_SLOT_IDENTITY, + correlationId: 'step_out_of_band', + eventData: { stepName: 'b-step', input: new Uint8Array() }, + }); + + await expect( + startStepLazily(runId, 'step_a', slotEventId(4)) + ).rejects.toThrow(SlotConflictError); + const eventId = await startStepLazily(runId, 'step_a', slotEventId(5)); + expect(eventId).toBe(slotEventId(5)); + await expect(slotsOf(runId)).resolves.toEqual([1, 2, 3, 4, 5]); + }); + it('reallocates around another instance holding the slot it picked', async () => { // Neither writer holds a log, so neither has anything to reconcile: the // loser takes the next free position instead of surfacing a conflict its From f9bd77c2deb8503ae9cd614d42bf743ff764d44c Mon Sep 17 00:00:00 2001 From: Peter Wielander Date: Fri, 31 Jul 2026 05:10:03 -0700 Subject: [PATCH 22/35] fix(core): send the instant a turbo start synthesizes its run from An optimistic start runs against a locally synthesized run row, so the workflow start time this invocation reports comes from the client clock while every later replay reads the persisted run. Sending that instant as the run_started event's occurredAt lets a backend record it as the run's startedAt, so the two agree instead of differing by the round-trip. Co-Authored-By: Claude Opus 5 --- .changeset/turbo-run-started-occurred-at.md | 5 +++ packages/core/src/runtime.test.ts | 36 +++++++++++++++++++++ packages/core/src/runtime.ts | 13 ++++++-- 3 files changed, 52 insertions(+), 2 deletions(-) create mode 100644 .changeset/turbo-run-started-occurred-at.md diff --git a/.changeset/turbo-run-started-occurred-at.md b/.changeset/turbo-run-started-occurred-at.md new file mode 100644 index 0000000000..bfbfbaf2d6 --- /dev/null +++ b/.changeset/turbo-run-started-occurred-at.md @@ -0,0 +1,5 @@ +--- +'@workflow/core': patch +--- + +Report the same workflow start time on an optimistically started run's first pass and its replays diff --git a/packages/core/src/runtime.test.ts b/packages/core/src/runtime.test.ts index 3a42435b01..8c2c6e4eec 100644 --- a/packages/core/src/runtime.test.ts +++ b/packages/core/src/runtime.test.ts @@ -15,6 +15,7 @@ import { import { ulid } from 'ulid'; import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest'; import { registerStepFunction } from './private.js'; +import { getWorkflowMetadata } from './step/get-workflow-metadata.js'; import { REPLAY_DIVERGENCE_MAX_RETRIES } from './runtime/constants.js'; import { setWorld } from './runtime/world.js'; import { workflowEntrypoint } from './runtime.js'; @@ -1463,6 +1464,17 @@ describe('workflowEntrypoint turbo mode', () => { return undefined; }); + // Records the workflow start time the step body observes, which is the one + // the synthesized run row carries under turbo. + let turboObservedStartedAt: Date | undefined; + registerStepFunction('turboMetadataStep', async () => { + turboObservedStartedAt = getWorkflowMetadata().workflowStartedAt; + return undefined; + }); + + const oneMetadataStepWorkflow = `const s = globalThis[Symbol.for("WORKFLOW_USE_STEP")]("turboMetadataStep"); + async function workflow() { return await s(); }${xform('workflow')}`; + const oneStepWorkflow = `const s = globalThis[Symbol.for("WORKFLOW_USE_STEP")]("turboStep"); async function workflow() { return await s(); }${xform('workflow')}`; @@ -1734,6 +1746,30 @@ describe('workflowEntrypoint turbo mode', () => { expect((redeliverRunStarted?.[2] as any)?.skipPreload).toBeUndefined(); }); + it('sends run_started the same instant it synthesizes the run from', async () => { + // Turbo starts the run against a locally synthesized run row, so the start + // time this invocation reports comes from the client clock. Backends that + // persist `occurredAt` record the run's `startedAt` from it, so sending it + // is what makes a later replay — which reads the persisted run — report the + // same `workflowStartedAt` this pass already captured into its steps. + turboObservedStartedAt = undefined; + const { handlerPromise, eventsCreate } = await driveTurbo({ + runId: 'wrun_turbo_occurred_at', + attempt: 1, + source: oneMetadataStepWorkflow, + specVersion: SPEC_VERSION_SLOT_IDENTITY, + }); + expect((await handlerPromise).status).toBe(204); + + const runStarted = eventsCreate.mock.calls.find( + (c) => (c[1] as any).eventType === 'run_started' + ); + const occurredAt = (runStarted?.[2] as { occurredAt?: Date } | undefined) + ?.occurredAt; + expect(occurredAt).toBeInstanceOf(Date); + expect(turboObservedStartedAt).toEqual(occurredAt); + }); + it('exits turbo (no forced optimistic) when the suspension creates a wait', async () => { const { handlerPromise, order } = await driveTurbo({ runId: 'wrun_turbo_wait', diff --git a/packages/core/src/runtime.ts b/packages/core/src/runtime.ts index 45928f36aa..19581650d2 100644 --- a/packages/core/src/runtime.ts +++ b/packages/core/src/runtime.ts @@ -1014,6 +1014,16 @@ export function workflowEntrypoint( // handler, optimistic step_started, terminal run writes) so // nothing is written before the run exists. recordRunStartedCreateStart(true); + // The instant this invocation calls the run started, sent + // with the event and reused for the synthesized run row + // below. Backends that persist `occurredAt` record the + // run's `startedAt` from it, which is what keeps + // `workflowStartedAt` identical between this optimistic + // pass and every later replay that reads the persisted + // run. Without it the two disagree by the round-trip, and + // a step's captured metadata no longer matches the + // workflow's on the next replay. + const now = new Date(); const startedPromise = world.events.create( runId, runStartedEvent, @@ -1024,7 +1034,7 @@ export function workflowEntrypoint( // run_started request the chained first step_started // waits on — shortening time-to-second-step — and the // wasted list+resolve it would otherwise compute. - { requestId, skipPreload: true } + { requestId, skipPreload: true, occurredAt: now } ); runReadyBarrier = startedPromise; // Turbo backgrounds run_started, so the non-turbo assignment @@ -1071,7 +1081,6 @@ export function workflowEntrypoint( if (usesSlotIdentity(runInput.specVersion)) { knownSlotFloor = FIRST_SLOT + 1; } - const now = new Date(); workflowRun = { runId, status: 'running', From 7d8f2f869f38eb18277f0872143d5fe25a37748a Mon Sep 17 00:00:00 2001 From: Peter Wielander Date: Fri, 31 Jul 2026 05:10:03 -0700 Subject: [PATCH 23/35] fix(core): send the instant a turbo start synthesizes its run from An optimistic start runs against a locally synthesized run row, so the workflow start time this invocation reports comes from the client clock while every later replay reads the persisted run. Sending that instant as the run_started event's occurredAt lets a backend record it as the run's startedAt, so the two agree instead of differing by the round-trip. Co-Authored-By: Claude Opus 5 --- .changeset/turbo-run-started-occurred-at.md | 5 +++ packages/core/src/runtime.test.ts | 36 +++++++++++++++++++++ packages/core/src/runtime.ts | 13 ++++++-- 3 files changed, 52 insertions(+), 2 deletions(-) create mode 100644 .changeset/turbo-run-started-occurred-at.md diff --git a/.changeset/turbo-run-started-occurred-at.md b/.changeset/turbo-run-started-occurred-at.md new file mode 100644 index 0000000000..bfbfbaf2d6 --- /dev/null +++ b/.changeset/turbo-run-started-occurred-at.md @@ -0,0 +1,5 @@ +--- +'@workflow/core': patch +--- + +Report the same workflow start time on an optimistically started run's first pass and its replays diff --git a/packages/core/src/runtime.test.ts b/packages/core/src/runtime.test.ts index 3a42435b01..8c2c6e4eec 100644 --- a/packages/core/src/runtime.test.ts +++ b/packages/core/src/runtime.test.ts @@ -15,6 +15,7 @@ import { import { ulid } from 'ulid'; import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest'; import { registerStepFunction } from './private.js'; +import { getWorkflowMetadata } from './step/get-workflow-metadata.js'; import { REPLAY_DIVERGENCE_MAX_RETRIES } from './runtime/constants.js'; import { setWorld } from './runtime/world.js'; import { workflowEntrypoint } from './runtime.js'; @@ -1463,6 +1464,17 @@ describe('workflowEntrypoint turbo mode', () => { return undefined; }); + // Records the workflow start time the step body observes, which is the one + // the synthesized run row carries under turbo. + let turboObservedStartedAt: Date | undefined; + registerStepFunction('turboMetadataStep', async () => { + turboObservedStartedAt = getWorkflowMetadata().workflowStartedAt; + return undefined; + }); + + const oneMetadataStepWorkflow = `const s = globalThis[Symbol.for("WORKFLOW_USE_STEP")]("turboMetadataStep"); + async function workflow() { return await s(); }${xform('workflow')}`; + const oneStepWorkflow = `const s = globalThis[Symbol.for("WORKFLOW_USE_STEP")]("turboStep"); async function workflow() { return await s(); }${xform('workflow')}`; @@ -1734,6 +1746,30 @@ describe('workflowEntrypoint turbo mode', () => { expect((redeliverRunStarted?.[2] as any)?.skipPreload).toBeUndefined(); }); + it('sends run_started the same instant it synthesizes the run from', async () => { + // Turbo starts the run against a locally synthesized run row, so the start + // time this invocation reports comes from the client clock. Backends that + // persist `occurredAt` record the run's `startedAt` from it, so sending it + // is what makes a later replay — which reads the persisted run — report the + // same `workflowStartedAt` this pass already captured into its steps. + turboObservedStartedAt = undefined; + const { handlerPromise, eventsCreate } = await driveTurbo({ + runId: 'wrun_turbo_occurred_at', + attempt: 1, + source: oneMetadataStepWorkflow, + specVersion: SPEC_VERSION_SLOT_IDENTITY, + }); + expect((await handlerPromise).status).toBe(204); + + const runStarted = eventsCreate.mock.calls.find( + (c) => (c[1] as any).eventType === 'run_started' + ); + const occurredAt = (runStarted?.[2] as { occurredAt?: Date } | undefined) + ?.occurredAt; + expect(occurredAt).toBeInstanceOf(Date); + expect(turboObservedStartedAt).toEqual(occurredAt); + }); + it('exits turbo (no forced optimistic) when the suspension creates a wait', async () => { const { handlerPromise, order } = await driveTurbo({ runId: 'wrun_turbo_wait', diff --git a/packages/core/src/runtime.ts b/packages/core/src/runtime.ts index 7e8bb8c915..6201e42b9f 100644 --- a/packages/core/src/runtime.ts +++ b/packages/core/src/runtime.ts @@ -1015,6 +1015,16 @@ export function workflowEntrypoint( // handler, optimistic step_started, terminal run writes) so // nothing is written before the run exists. recordRunStartedCreateStart(true); + // The instant this invocation calls the run started, sent + // with the event and reused for the synthesized run row + // below. Backends that persist `occurredAt` record the + // run's `startedAt` from it, which is what keeps + // `workflowStartedAt` identical between this optimistic + // pass and every later replay that reads the persisted + // run. Without it the two disagree by the round-trip, and + // a step's captured metadata no longer matches the + // workflow's on the next replay. + const now = new Date(); const startedPromise = world.events.create( runId, runStartedEvent, @@ -1025,7 +1035,7 @@ export function workflowEntrypoint( // run_started request the chained first step_started // waits on — shortening time-to-second-step — and the // wasted list+resolve it would otherwise compute. - { requestId, skipPreload: true } + { requestId, skipPreload: true, occurredAt: now } ); runReadyBarrier = startedPromise; // Turbo backgrounds run_started, so the non-turbo assignment @@ -1072,7 +1082,6 @@ export function workflowEntrypoint( if (usesSlotIdentity(runInput.specVersion)) { knownSlotFloor = FIRST_SLOT + 1; } - const now = new Date(); workflowRun = { runId, status: 'running', From 48084a7085485775e7606c061b1f82fe71fe1dde Mon Sep 17 00:00:00 2001 From: Peter Wielander Date: Fri, 31 Jul 2026 06:01:09 -0700 Subject: [PATCH 24/35] fix(core): render structured error messages and name what a divergence waited for MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Three diagnostic gaps that together made replay divergence unreadable: - `composeLogLine` dropped `errorMessage` whenever the message did not already contain it, so a warn carrying an error alongside its own summary line logged the symptom and none of the diagnosis. - An unconsumable event named only itself. It is almost always an event whose entity this replay never issued, so the pending invocation queue is what distinguishes "never issued" from "issued under another id". - An inline step batch abandoned on a fenced claim logged neither which member was fenced nor how the others settled. The fence is per-write, so a batch can split: the rejected claim writes nothing while a sibling on a different slot commits. Also read a failed run's error through `returnValue()` in the race-repro harness — `runs.get` returns the raw serialized payload, so every corruption in the report carried a code and no message. Co-Authored-By: Claude Opus 5 --- .changeset/tidy-moons-observe.md | 5 +++ .../core/e2e/event-log-race-repro.test.ts | 35 +++++++++++++++++-- packages/core/src/log-format.test.ts | 26 +++++++++++++- packages/core/src/log-format.ts | 20 ++++++++--- packages/core/src/runtime.ts | 26 ++++++++++++-- packages/core/src/workflow.ts | 8 ++++- 6 files changed, 110 insertions(+), 10 deletions(-) create mode 100644 .changeset/tidy-moons-observe.md diff --git a/.changeset/tidy-moons-observe.md b/.changeset/tidy-moons-observe.md new file mode 100644 index 0000000000..9c6c5b7692 --- /dev/null +++ b/.changeset/tidy-moons-observe.md @@ -0,0 +1,5 @@ +--- +'@workflow/core': patch +--- + +Name the divergent event's pending invocations and the fenced member of an inline step batch in replay-divergence logs diff --git a/packages/core/e2e/event-log-race-repro.test.ts b/packages/core/e2e/event-log-race-repro.test.ts index e98f6fd10a..b40fed2ec3 100644 --- a/packages/core/e2e/event-log-race-repro.test.ts +++ b/packages/core/e2e/event-log-race-repro.test.ts @@ -373,6 +373,30 @@ function validateStormReturn(value: unknown): { return { stragglers }; } +/** + * Reads a terminal-failed run's error through `returnValue()`, which hydrates + * the stored payload into an Error. Returns undefined when the read itself + * fails — the outcome is already known from `errorCode`, so a missing message + * degrades the report rather than the classification. + */ +async function readFailureMessage( + run: Run +): Promise<{ name?: string; message?: string } | undefined> { + try { + await run.returnValue(); + return undefined; + } catch (err) { + if (WorkflowRunFailedError.is(err)) { + const cause = err.cause; + return { + name: cause instanceof Error ? cause.name : err.name, + message: cause instanceof Error ? cause.message : err.message, + }; + } + return undefined; + } +} + async function pollTerminalRun( run: Run, startedAt: number, @@ -412,13 +436,20 @@ async function pollTerminalRun( errorCode?: string; error?: { name?: string; message?: string }; }; + // `runs.get` hands back the raw serialized error payload, not an Error, so + // reading `.message` off it yields undefined and the report records the + // code with no diagnosis. Read the failure through the public + // return-value path, which hydrates it. For a corruption that message + // carries the divergent event and what the replay was waiting for, which + // is the whole reason to keep the report. + const hydrated = await readFailureMessage(run); return { ...base, outcome: classifyFailure(failure.errorCode), status: runData.status, errorCode: failure.errorCode, - errorMessage: failure.error?.message, - errorName: failure.error?.name, + errorMessage: hydrated?.message ?? failure.error?.message, + errorName: hydrated?.name ?? failure.error?.name, durationMs: Date.now() - startedAt, }; } diff --git a/packages/core/src/log-format.test.ts b/packages/core/src/log-format.test.ts index c673893f06..3b78208744 100644 --- a/packages/core/src/log-format.test.ts +++ b/packages/core/src/log-format.test.ts @@ -113,6 +113,29 @@ describe('composeLogLine', () => { `); }); + test('renders errorMessage when the message does not already carry it', () => { + // The replay-divergence warn writes its own summary line and passes the + // error only as metadata, so this is the sole place the divergent event's + // identity appears. Dropping it leaves the log naming a symptom with no + // way to tell which event diverged. + const out = composeLogLine( + PREFIX, + 'Workflow replay diverged; queueing a recovery replay before declaring the event log corrupted', + { + errorCode: 'REPLAY_DIVERGENCE', + errorMessage: + 'Replay could not consume event: eventType=step_created, correlationId=step_00000000000000000000000025.', + divergenceCount: 1, + } + ); + expect(out).toMatchInlineSnapshot(` + "[workflow-sdk] Workflow replay diverged; queueing a recovery replay before declaring the event log corrupted + code REPLAY_DIVERGENCE + error Replay could not consume event: eventType=step_created, correlationId=step_00000000000000000000000025. + divergenceCount 1" + `); + }); + test('falls back gracefully on machine names it cannot parse', () => { const out = composeLogLine(PREFIX, 'msg', { workflowRunId: 'wrun_X', @@ -156,7 +179,8 @@ describe('composeLogLine', () => { user error · Error run wrun_01ABC · myWorkflow (./workflows/x) step step_01XYZ · add (./workflows/x) - retry 4 attempts · 3 max retries" + retry 4 attempts · 3 max retries + error Transient failure" `); }); }); diff --git a/packages/core/src/log-format.ts b/packages/core/src/log-format.ts index d63e847e3f..b3daa3263c 100644 --- a/packages/core/src/log-format.ts +++ b/packages/core/src/log-format.ts @@ -36,7 +36,7 @@ export function composeLogLine( ): string { const [framing, ...rest] = message.split('\n'); const body = rest.join('\n'); - const fields = renderStructuredFields(framing ?? '', metadata); + const fields = renderStructuredFields(message, metadata); const trimmedBody = trimStackBody(body); const lines: string[] = [`${prefix} ${framing ?? ''}`]; @@ -46,19 +46,21 @@ export function composeLogLine( } function renderStructuredFields( - framing: string, + message: string, metadata: Record | undefined ): string | null { if (!metadata || Object.keys(metadata).length === 0) return null; // Drop fields that the message already encodes. We render framings and // stacks into the message string itself in step executor / combined runtime, so - // repeating them here would be pure noise. + // repeating them here would be pure noise. The whole message counts, not just + // its first line: callers that pass `${framing}\n${stack}` put the error's + // text in the stack's leading `Name: message` line. const redundant = new Set(); redundant.add('errorStack'); if ( typeof metadata.errorMessage === 'string' && - framing.includes(metadata.errorMessage as string) + message.includes(metadata.errorMessage as string) ) { redundant.add('errorMessage'); } @@ -130,6 +132,16 @@ function renderStructuredFields( lines.push(` ${kvKey('code')} ${Ansi.dim(errorCode)}`); } + // The message only duplicates the framing when the framing was built from + // the error itself (step executor, terminal run failures), and that case is + // already marked redundant above. Everywhere else — a warn that carries an + // error alongside its own summary line — this is the only place the error's + // own text appears, so dropping it loses the diagnosis. + const errorMessage = pickString(metadata, 'errorMessage'); + if (errorMessage && !redundant.has('errorMessage')) { + lines.push(` ${kvKey('error')} ${errorMessage}`); + } + const hint = pickString(metadata, 'hint'); if (hint) { lines.push(` ${Ansi.hint(hint)}`); diff --git a/packages/core/src/runtime.ts b/packages/core/src/runtime.ts index 6201e42b9f..2559312439 100644 --- a/packages/core/src/runtime.ts +++ b/packages/core/src/runtime.ts @@ -2499,10 +2499,32 @@ export function workflowEntrypoint( // the sibling executions to settle first so no owned // body is in flight when the ack path runs. if (requiresFreshReplay(stepErr)) { - await Promise.allSettled(stepExecutionPromises); + const settled = await Promise.allSettled( + stepExecutionPromises + ); runtimeLogger.warn( 'Inline step claim rejected as stale; re-invoking run for a fresh replay', - { workflowRunId: runId, loopIteration } + { + workflowRunId: runId, + loopIteration, + // Which members of the batch were fenced and + // which committed. A fence is per-write, so a + // batch can be split: the rejected claim wrote + // nothing, but a sibling holding a different + // slot may have committed its step. That + // asymmetry is the shape to look for when a + // later replay cannot consume a step event. + batchSteps: inlineExecutions + .map( + (s, i) => + `${s.correlationId}:${settled[i]?.status === 'rejected' ? 'rejected' : 'settled'}` + ) + .join(', '), + errorMessage: + stepErr instanceof Error + ? stepErr.message + : String(stepErr), + } ); // The finally below resumes the replay budget // before this return completes. diff --git a/packages/core/src/workflow.ts b/packages/core/src/workflow.ts index 97c9acee99..9fb568d955 100644 --- a/packages/core/src/workflow.ts +++ b/packages/core/src/workflow.ts @@ -213,9 +213,15 @@ export async function runWorkflow( updateTimestamp(+event.createdAt); }, onUnconsumedEvent: (event) => { + // Name what the replay was waiting for instead. An unconsumable event + // is almost always one whose entity this replay never issued, or + // issued under a different correlation ID; the pending invocation + // queue is the only place that distinction is visible, and without it + // the log names a symptom with no way to reach the cause. + const pending = [...workflowContext.invocationsQueue.keys()]; workflowDiscontinuation.reject( new ReplayDivergenceError( - `Replay could not consume event: eventType=${event.eventType}, correlationId=${event.correlationId}, eventId=${event.eventId}.`, + `Replay could not consume event: eventType=${event.eventType}, correlationId=${event.correlationId}, eventId=${event.eventId}. Pending invocations: ${pending.length > 0 ? pending.join(', ') : '(none)'}.`, { eventId: event.eventId } ) ); From 0f2677023eb56b7817971924d99a816f44a1fd62 Mon Sep 17 00:00:00 2001 From: Peter Wielander Date: Fri, 31 Jul 2026 06:34:59 -0700 Subject: [PATCH 25/35] fix(core): reclaim a lost inline step slot instead of splitting the batch MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit An inline step's step_started claim is fenced per-write under slot identity, so a 409 only proves another writer took that write's number — routinely true, since the backend allocates outside events from the same next-free pointer the client reserves from. Abandoning the whole batch on it left the loser's events landing seconds later, after a whole later phase, in an order no single replay could consume. stepClaimFence keeps a watermark-guarded run on its single shared fence (a 412 does mean the view is stale, and the batch is meant to fail as a unit) and gives a slot-numbered run an in-place reclaim: merge the delta, reserve past it, re-claim. The reservation pointer is now absolute and only moves forward, so a merge cannot hand the retrying writer a slot a sibling is still in flight on. --- .changeset/inline-claim-reclaim.md | 5 + packages/core/src/logger.test.ts | 4 +- packages/core/src/runtime.ts | 35 +++-- packages/core/src/runtime/helpers.test.ts | 116 +++++++++++++++-- packages/core/src/runtime/helpers.ts | 121 +++++++++++++---- .../core/src/runtime/step-executor.test.ts | 2 +- packages/core/src/runtime/step-executor.ts | 123 ++++++++++-------- 7 files changed, 307 insertions(+), 99 deletions(-) create mode 100644 .changeset/inline-claim-reclaim.md diff --git a/.changeset/inline-claim-reclaim.md b/.changeset/inline-claim-reclaim.md new file mode 100644 index 0000000000..9d57541efc --- /dev/null +++ b/.changeset/inline-claim-reclaim.md @@ -0,0 +1,5 @@ +--- +'@workflow/core': patch +--- + +Keep a batch of inline steps together when one of its event writes loses a race, instead of discarding the batch diff --git a/packages/core/src/logger.test.ts b/packages/core/src/logger.test.ts index 5560c9e213..78066923ca 100644 --- a/packages/core/src/logger.test.ts +++ b/packages/core/src/logger.test.ts @@ -148,6 +148,7 @@ describe('logger', () => { user error · FatalError run wrun_123 step step_456 + error boom hint: Move the call to a step function.", ], ] @@ -178,7 +179,8 @@ describe('logger', () => { user error · Error run wrun_abc step step_xyz - retry 4 attempts · 3 max retries", + retry 4 attempts · 3 max retries + error Transient failure", ], ] `); diff --git a/packages/core/src/runtime.ts b/packages/core/src/runtime.ts index 2559312439..648f01cd70 100644 --- a/packages/core/src/runtime.ts +++ b/packages/core/src/runtime.ts @@ -52,6 +52,7 @@ import { countStepStartedEvents } from './runtime/count-step-started-events.js'; import { appendUniqueEvents, eventCreateFenceFor, + stepClaimFence, getQueueOverhead, getWorkflowQueueName, handleHealthCheckMessage, @@ -2188,10 +2189,14 @@ export function workflowEntrypoint( // (retried over the reloaded log, or exhausted into // a queue re-invocation), AND the lazy step_started // claim of its next inline step, which is fenced too - // (threaded below via - // `eventCreateFence`; on rejection the batch is - // abandoned and re-invoked for a fresh replay, so a - // stale view can never commit a step). Hooks created + // (threaded below via `stepClaimFence`; on rejection + // the batch is abandoned and re-invoked for a fresh + // replay, so a stale view can never commit a step). + // A slot-numbered run gets there differently — the + // claim merges the missed events and retries in + // place, so the same events are observed without + // discarding the batch. See stepClaimFence. + // Hooks created // by THIS suspension are inside the delta (their // `hook_created` lands before the step-terminal // write), so only their `hook_received` responses @@ -2338,8 +2343,8 @@ export function workflowEntrypoint( // could claim — and commit — a step scheduled off a view // that misses an out-of-band event. One log for the // whole batch so each claim draws its own event slot; - // `eventCreateFenceFor` yields undefined for a run - // fenced neither way, leaving those claims as they were. + // `stepClaimFence` leaves the claims of a run fenced + // neither way exactly as they were. // // The suspension's own log, not a second one over the // same snapshot: its reservations are what the hook and @@ -2360,7 +2365,8 @@ export function workflowEntrypoint( // positional, so it has to be assigned before these // executions start racing each other, and the order // it is assigned in has to be replay-stable. - const eventCreateFence = eventCreateFenceFor( + const claimFence = stepClaimFence( + runId, inlineClaimLog, workflowRun.specVersion, { @@ -2446,7 +2452,7 @@ export function workflowEntrypoint( // see suppressOptimisticStart above. suppressOptimisticStart, runReadyBarrier, - eventCreateFence, + claimFence, ...(stepIndex === 0 && s.lazyStepInput !== undefined && latencyTracking @@ -2488,12 +2494,15 @@ export function workflowEntrypoint( ); } catch (stepErr) { // An incomplete-view rejection of an inline - // step_started claim (412 stale watermark, or 409 - // taken slot): the loaded view this batch was + // step_started claim: the loaded view this batch was // scheduled from is behind an out-of-band event (e.g. - // a received hook), so the claim was fenced by the - // guard and no step events were written. Abandon the - // batch — any optimistic body result is discarded by + // a received hook), so the claim was fenced and no + // step events were written. Under the watermark that + // is every 412; under slot identity `stepClaimFence` + // first merges the missed events and re-claims in + // place, so a 409 only arrives here once those + // retries are exhausted. Abandon the batch — any + // optimistic body result is discarded by // executeStep's reconciliation — and re-invoke for a // fresh replay that observes the new event. Wait for // the sibling executions to settle first so no owned diff --git a/packages/core/src/runtime/helpers.test.ts b/packages/core/src/runtime/helpers.test.ts index 16374fba70..9cf79dcdfc 100644 --- a/packages/core/src/runtime/helpers.test.ts +++ b/packages/core/src/runtime/helpers.test.ts @@ -34,6 +34,7 @@ import { requiresFreshReplay, reserveSlot, stateUpdatedAtForCreate, + stepClaimFence, toMutableEventLog, withEventCreateFence, withPreconditionRetry, @@ -530,7 +531,7 @@ describe('slot bookkeeping', () => { // element is not necessarily its newest event. const log = toMutableEventLog([slotEvent(3), slotEvent(1)], 'c0'); expect(log.maxSlot).toBe(3); - expect(log.reserved).toBe(0); + expect(log.nextSlot).toBe(4); }); it('reports maxSlot 0 for an empty or ULID-numbered log', () => { @@ -567,21 +568,33 @@ describe('slot bookkeeping', () => { expect(log.events).toHaveLength(3); }); - it('raises maxSlot and drops reservations when a newer delta is merged in', () => { + it('raises the reservation pointer past a newer delta', () => { const log = toMutableEventLog([slotEvent(1)], 'c0'); reserveSlot(log); reserveSlot(log); - expect(log.reserved).toBe(2); + expect(log.nextSlot).toBe(4); mergeLoadedEvents(log, [slotEvent(2), slotEvent(5)]); expect(log.maxSlot).toBe(5); - // The merged events are the authority on which slots are taken, so the - // outstanding reservations (slots 2 and 3) are void. - expect(log.reserved).toBe(0); expect(reserveSlot(log)).toBe(6); }); + it('never rewinds the reservation pointer onto an outstanding slot', () => { + // A writer that loses its slot merges the delta and reserves again while + // its siblings are still in flight on theirs. Rewinding to `maxSlot + 1` + // would hand it slot 4, which a sibling already holds. + const log = toMutableEventLog([slotEvent(1)], 'c0'); + expect(reserveSlot(log)).toBe(2); + expect(reserveSlot(log)).toBe(3); + expect(reserveSlot(log)).toBe(4); + + mergeLoadedEvents(log, [slotEvent(2)]); + + expect(log.maxSlot).toBe(2); + expect(reserveSlot(log)).toBe(5); + }); + it('deduplicates merged events by id', () => { const log = toMutableEventLog([slotEvent(1)], 'c0'); mergeLoadedEvents(log, [slotEvent(1), slotEvent(2)]); @@ -640,7 +653,7 @@ describe('slot bookkeeping', () => { eventCreateFenceFor(log, SPEC_VERSION_SLOT_IDENTITY - 1, { extraEvents: 1, }); - expect(log.reserved).toBe(0); + expect(log.nextSlot).toBe(1); }); it('proposes no event id for a ULID-numbered run', () => { @@ -648,7 +661,7 @@ describe('slot bookkeeping', () => { const log = toMutableEventLog([], null); const fence = eventCreateFenceFor(log, SPEC_VERSION_SLOT_IDENTITY - 1); expect(fence?.eventId).toBeUndefined(); - expect(log.reserved).toBe(0); + expect(log.nextSlot).toBe(1); }); }); @@ -860,6 +873,93 @@ describe('withEventCreateFence', () => { }); }); +describe('stepClaimFence', () => { + const slotEvent = (slot: number) => makeEvent(slotEventId(slot)); + + beforeEach(() => { + eventsListMock.mockReset(); + }); + + it('numbers a batch in the order its claims were built, not the order they fire', async () => { + // The batch is built during replay and only starts racing afterwards, so + // the slot of each member has to be fixed at build time for the numbering + // to be replay-stable. + const log = toMutableEventLog([slotEvent(1)], 'c0'); + const first = stepClaimFence('wrun_test', log, SPEC_VERSION_SLOT_IDENTITY, { + extraEvents: 1, + }); + const second = stepClaimFence('wrun_test', log, SPEC_VERSION_SLOT_IDENTITY); + + const claimed: (string | undefined)[] = []; + const record = (fence?: { eventId?: string }) => { + claimed.push(fence?.eventId); + return Promise.resolve('ok'); + }; + // Fired in reverse: the numbering must not depend on it. + await second(record); + await first(record); + + expect(claimed).toEqual([slotEventId(4), slotEventId(3)]); + }); + + it('reclaims a lost slot in place, keeping the batch adjacent', async () => { + // The server allocates an outside event from the same next-free pointer + // the client reserves from, so losing a claim is routine. Abandoning the + // batch on it would leave this step's events far later in the log than its + // siblings' — an order no single replay can consume — and leave the lost + // slot permanently empty. + const log = toMutableEventLog([slotEvent(1)], 'c0'); + const claim = stepClaimFence('wrun_test', log, SPEC_VERSION_SLOT_IDENTITY); + const sibling = stepClaimFence( + 'wrun_test', + log, + SPEC_VERSION_SLOT_IDENTITY + ); + + const claimed: string[] = []; + const op = vi.fn(async (fence?: { eventId?: string }) => { + claimed.push(fence?.eventId as string); + if (claimed.length === 1) { + // An out-of-band hook took slot 2 while the batch was being built. + throw new SlotConflictError('taken', { + eventId: fence?.eventId as string, + events: [slotEvent(2)], + cursor: 'c1', + }); + } + return 'done'; + }); + + await expect(claim(op)).resolves.toBe('done'); + await expect(sibling(async (f) => f?.eventId)).resolves.toBe( + slotEventId(3) + ); + // Reclaimed above the merged event rather than propagating the conflict. + expect(claimed).toEqual([slotEventId(2), slotEventId(4)]); + }); + + it('leaves a ULID-numbered batch on one shared watermark, unretried', async () => { + // A 412 compares time, so every member of the batch carries the same fence + // value and the batch fails as a unit — which is what the caller's + // fresh-replay path expects. + const time = 1_700_000_000_000; + const log = toMutableEventLog([makeUlidEvent(time)], 'c0'); + const claim = stepClaimFence( + 'wrun_test', + log, + SPEC_VERSION_SLOT_IDENTITY - 1 + ); + const op = vi.fn(async () => { + throw new PreconditionFailedError('stale'); + }); + + await expect(claim(op)).rejects.toBeInstanceOf(PreconditionFailedError); + expect(op).toHaveBeenCalledTimes(1); + expect(op).toHaveBeenCalledWith({ stateUpdatedAt: time }); + expect(eventsListMock).not.toHaveBeenCalled(); + }); +}); + describe('requiresFreshReplay', () => { it('covers both fences, so neither numbering fails the run', () => { // Each fence reports an incomplete view in its own dialect. A caller that diff --git a/packages/core/src/runtime/helpers.ts b/packages/core/src/runtime/helpers.ts index 407f202635..c1a9248f32 100644 --- a/packages/core/src/runtime/helpers.ts +++ b/packages/core/src/runtime/helpers.ts @@ -662,11 +662,11 @@ export interface MutableEventLog { */ maxSlot: number; /** - * Slots handed out by `reserveSlot` past `maxSlot` whose events have not been - * merged back yet. Reset whenever the log is merged into, because the merged - * events are the authority on which slots are taken. + * Next slot `reserveSlot` will hand out. Absolute, and it only ever moves + * forward: a merge can raise it past the events it brought in, but must never + * lower it onto a slot already handed to a writer that is still in flight. */ - reserved: number; + nextSlot: number; } /** @@ -683,17 +683,18 @@ export function toMutableEventLog( cursor: string | null, slotFloor = 0 ): MutableEventLog { + const maxSlot = Math.max(maxSlotOf(events), slotFloor); return { events, cursor, - maxSlot: Math.max(maxSlotOf(events), slotFloor), - reserved: 0, + maxSlot, + nextSlot: maxSlot + 1, }; } /** * Merges loaded events into `log` in place, keeping `maxSlot` current and - * dropping outstanding reservations (the merged events supersede them). + * advancing the reservation pointer past the events merged in. */ export function mergeLoadedEvents( log: MutableEventLog, @@ -701,7 +702,7 @@ export function mergeLoadedEvents( ): void { appendUniqueEvents(log.events, events); log.maxSlot = Math.max(log.maxSlot, maxSlotOf(events)); - log.reserved = 0; + log.nextSlot = Math.max(log.nextSlot, log.maxSlot + 1); } /** @@ -713,10 +714,16 @@ export function mergeLoadedEvents( * in the flush would propose the same slot and all but one would conflict, on * every single flush. Operations are built in deterministic replay order, so * the slot each one draws is replay-stable too. + * + * The pointer is never rewound by a merge, only pushed forward. A writer that + * loses its slot merges the delta and reserves again while its siblings still + * hold theirs; rewinding to `maxSlot + 1` would hand it a sibling's slot and + * turn one conflict into a chain of them. */ export function reserveSlot(log: MutableEventLog): number { - log.reserved += 1; - return log.maxSlot + log.reserved; + const slot = log.nextSlot; + log.nextSlot = slot + 1; + return slot; } /** @@ -859,7 +866,7 @@ export interface EventCreateFence { /** * The fence for a create that is deliberately **not** retried in place, because * a rejection means the committed decision itself is stale and only a fresh - * replay can revise it (`run_completed`, an inline `step_started` claim). + * replay can revise it (`run_completed`). * * Claims a slot off `log` for a slot-numbered run — which counts as a * reservation, so a caller that fences several creates from one log gets a @@ -885,19 +892,80 @@ export function eventCreateFenceFor( options?: { extraEvents?: number } ): EventCreateFence | undefined { if (usesSlotIdentity(specVersion)) { - const maxSlot = log.maxSlot; - // The extra events sit below the one being created, matching the order a - // reader expects (a step is created before it starts) — so their slots are - // reserved first and the claim names the last of the run. - for (let i = 0; i < (options?.extraEvents ?? 0); i++) { - reserveSlot(log); - } - return { eventId: slotEventId(reserveSlot(log)), maxSlot }; + return reserveSlotFence(log, options?.extraEvents ?? 0); } const stateUpdatedAt = stateUpdatedAtForCreate(log.events, specVersion); return stateUpdatedAt !== undefined ? { stateUpdatedAt } : undefined; } +/** + * Reserves this write's slots off `log` and names the one the event itself + * takes. + * + * `extraEvents` sit below the one being created, matching the order a reader + * expects (a step is created before it starts), so their slots are reserved + * first and the claim names the last of the run — a World that writes a pair + * derives the lower id from the one it was given. + */ +function reserveSlotFence( + log: MutableEventLog, + extraEvents: number +): EventCreateFence { + const maxSlot = log.maxSlot; + for (let i = 0; i < extraEvents; i++) { + reserveSlot(log); + } + return { eventId: slotEventId(reserveSlot(log)), maxSlot }; +} + +/** + * Runs one event create under whichever fence its run uses, handling a lost + * claim however that run's scheme requires. + */ +export type FencedCreate = ( + op: (fence: EventCreateFence | undefined) => Promise +) => Promise; + +/** + * The fence for an inline step's `step_started` claim. + * + * A slot-numbered run retries a lost claim in place; a watermark-guarded run + * does not, and lets the rejection abandon the batch for a fresh replay. The + * asymmetry is in what a rejection proves, and it is the difference between a + * batch that stays contiguous and one that splits: + * + * - A 412 compares the *time* of the newest outside event, so every claim in a + * batch carries the same fence value and a stale view fails all of them. The + * batch is abandoned as a unit, nothing is written, and the fresh replay + * reschedules from a complete view. + * - A 409 only proves another writer took this write's *number*. That happens + * routinely without any staleness: the server allocates outside events from + * the same next-free pointer the client reserves from, so any outside event + * landing mid-batch takes the slot the batch's next claim is holding. Fencing + * the batch on it splits it — the loser writes nothing while its siblings + * commit, and the loser's events land far later in the log (or never), leaving + * an order no single replay can consume. Taking another number instead keeps + * the batch's events adjacent and its slots dense. + */ +export function stepClaimFence( + runId: string, + log: MutableEventLog, + specVersion: number | undefined, + options?: { extraEvents?: number } +): FencedCreate { + if (usesSlotIdentity(specVersion)) { + // Reserved here, synchronously, rather than when the claim fires: a batch's + // claims have to be numbered in replay order, and they only start racing + // each other afterwards. Retries re-reserve at that point by necessity — + // the merged delta has moved the log — but by then this write is the only + // one of the batch still choosing a slot. + const initialFence = reserveSlotFence(log, options?.extraEvents ?? 0); + return (op) => withSlotRetry(runId, log, op, { ...options, initialFence }); + } + const fence = eventCreateFenceFor(log, specVersion, options); + return (op) => op(fence); +} + /** * Runs a replay-context event creation that claims its own event slot. * @@ -917,15 +985,20 @@ export function eventCreateFenceFor( export async function withSlotRetry( runId: string, log: MutableEventLog, - op: (fence: EventCreateFence) => Promise + op: (fence: EventCreateFence) => Promise, + options?: { extraEvents?: number; initialFence?: EventCreateFence } ): Promise { for (let attempt = 0; ; attempt++) { // Claimed per attempt, not once up front: a merged delta moves the log's - // high-water mark, so the previous claim is stale by definition. - const maxSlot = log.maxSlot; - const eventId = slotEventId(reserveSlot(log)); + // high-water mark, so the previous claim is stale by definition. The first + // attempt can carry a slot the caller reserved earlier, for a caller whose + // numbering has to be assigned in a particular order (see stepClaimFence). + const fence = + (attempt === 0 ? options?.initialFence : undefined) ?? + reserveSlotFence(log, options?.extraEvents ?? 0); + const eventId = fence.eventId; try { - return await op({ eventId, maxSlot }); + return await op(fence); } catch (error) { if ( !SlotConflictError.is(error) || diff --git a/packages/core/src/runtime/step-executor.test.ts b/packages/core/src/runtime/step-executor.test.ts index 2444358b37..e2d0869285 100644 --- a/packages/core/src/runtime/step-executor.test.ts +++ b/packages/core/src/runtime/step-executor.test.ts @@ -191,7 +191,7 @@ describe('executeStep — compute instance stamping', () => { workflowStartedAt: Date.now(), stepId, stepName, - eventCreateFence: { stateUpdatedAt: 1_700_000_000_000 }, + claimFence: (op) => op({ stateUpdatedAt: 1_700_000_000_000 }), }); const started = createSpy.mock.calls.filter( diff --git a/packages/core/src/runtime/step-executor.ts b/packages/core/src/runtime/step-executor.ts index fdfa6358af..0417a5b371 100644 --- a/packages/core/src/runtime/step-executor.ts +++ b/packages/core/src/runtime/step-executor.ts @@ -49,7 +49,11 @@ import { isOptimisticInlineStartExplicitlyDisabled, } from './constants.js'; import { getPortLazy } from './get-port-lazy.js'; -import { type EventCreateFence, memoizeEncryptionKey } from './helpers.js'; +import { + type EventCreateFence, + type FencedCreate, + memoizeEncryptionKey, +} from './helpers.js'; import { computeStepLatencyEventData, type StepLatencyEventData, @@ -135,23 +139,27 @@ export interface StepExecutorParams { */ inlineDeltaSinceCursor?: string; /** - * Concurrency fence to attach to this step's `step_started` claim: the event - * slot the claim occupies, or the caller's replay snapshot (`stateUpdatedAt`, - * epoch ms of the latest event it loaded) for a run on the older numbering. + * Runs this step's `step_started` claim under its run's concurrency fence: + * the event slot the claim occupies, or the caller's replay snapshot + * (`stateUpdatedAt`, epoch ms of the latest event it loaded) for a run on the + * older numbering. * * On the lazy inline path the claim is the step's FIRST durable write (its * `step_created` is deferred), so without a fence it would be unguarded * entirely: a replay working from a stale view could claim — and then commit — * a step scheduled without observing an out-of-band event. A fencing World * rejects such a claim with `SlotConflictError` (409) or - * `PreconditionFailedError` (412); executeStep does NOT translate either - * rejection (re-claiming in place would still commit the stale schedule), so - * it propagates for the caller to abandon the batch and force a fresh replay. + * `PreconditionFailedError` (412). + * + * Whether a rejection is retried in place is the caller's decision, made per + * scheme — see `stepClaimFence`. Either way executeStep does NOT translate a + * rejection that reaches it, so an unretried one propagates for the caller to + * abandon the batch and force a fresh replay. * * Undefined when the caller has no snapshot, or when the watermark guard is * disabled on a run that uses it; Worlds that fence neither way ignore it. */ - eventCreateFence?: EventCreateFence; + claimFence?: FencedCreate; /** * Suppress optimistic inline start for this step regardless of * `WORKFLOW_OPTIMISTIC_INLINE_START` / `forceOptimisticStart`: take the @@ -259,6 +267,10 @@ export async function executeStep( stepName, } = params; const isVercel = process.env.VERCEL_URL !== undefined; + // Unfenced when the caller passes no fence — every World that fences + // ignores the field it does not understand, so this is the same create it + // was before either mechanism existed. + const runClaim: FencedCreate = params.claimFence ?? ((op) => op(undefined)); // Gate payload compression on the run's specVersion. const compression = (params.runSpecVersion ?? 0) >= SPEC_VERSION_SUPPORTS_COMPRESSION; @@ -515,11 +527,13 @@ export async function executeStep( let step: Step; // Params for the `step_started` create on either path below: the ambient - // compute-instance stamp plus the claim fence. - const startEventParams: CreateEventParams = { + // compute-instance stamp plus whichever fence the claim is running under. + const startEventParams = ( + fence: EventCreateFence | undefined + ): CreateEventParams => ({ computeInstanceId: COMPUTE_INSTANCE_ID, - ...params.eventCreateFence, - }; + ...fence, + }); // `Date.now()` taken immediately before the `step_started` create is // issued (either path below) — anchors RSFS's end point. See // StepLatencyEventData.rsfs and the call sites below. @@ -559,27 +573,29 @@ export async function executeStep( // RSFS measures the run_started-to-POST stretch, and the barrier // wait IS part of that stretch under turbo. stepStartPostSentAtMs = Date.now(); - return world.events.create( - workflowRunId, - { - eventType: 'step_started', - specVersion: SPEC_VERSION_CURRENT, - correlationId: stepId, - eventData: { - stepName, - workflowName, - input: params.lazyStepInput, - // Inline-ownership stamp — see StepExecutorParams.ownerMessageId. - ...(params.ownerMessageId !== undefined - ? { ownerMessageId: params.ownerMessageId } - : {}), + // Fence the claim — see StepExecutorParams.claimFence. A rejection + // the fence does not retry surfaces via reconcileOptimisticStart as a + // non-translatable error: the body result is discarded and the + // rejection propagates to the caller. + return runClaim((fence) => + world.events.create( + workflowRunId, + { + eventType: 'step_started', + specVersion: SPEC_VERSION_CURRENT, + correlationId: stepId, + eventData: { + stepName, + workflowName, + input: params.lazyStepInput, + // Inline-ownership stamp — see StepExecutorParams.ownerMessageId. + ...(params.ownerMessageId !== undefined + ? { ownerMessageId: params.ownerMessageId } + : {}), + }, }, - }, - // Fence the claim — see StepExecutorParams.eventCreateFence. A 409/412 - // rejection surfaces via reconcileOptimisticStart as a - // non-translatable error: the body result is discarded and the - // rejection propagates to the caller. - startEventParams + startEventParams(fence) + ) ); } ); @@ -620,26 +636,29 @@ export async function executeStep( ? { ownerMessageId: params.ownerMessageId } : {}; stepStartPostSentAtMs = Date.now(); - const startResult = await world.events.create( - workflowRunId, - { - eventType: 'step_started', - specVersion: SPEC_VERSION_CURRENT, - correlationId: stepId, - eventData: - params.lazyStepInput !== undefined - ? { - stepName, - workflowName, - input: params.lazyStepInput, - ...ownershipStamp, - } - : { stepName, ...ownershipStamp }, - }, - // Fence the claim — see StepExecutorParams.eventCreateFence. A 409/412 - // rejection is intentionally NOT translated by startErrorToResult - // below, so it propagates to the caller for a fresh replay. - startEventParams + // Fence the claim — see StepExecutorParams.claimFence. A rejection the + // fence does not retry is intentionally NOT translated by + // startErrorToResult below, so it propagates to the caller for a fresh + // replay. + const startResult = await runClaim((fence) => + world.events.create( + workflowRunId, + { + eventType: 'step_started', + specVersion: SPEC_VERSION_CURRENT, + correlationId: stepId, + eventData: + params.lazyStepInput !== undefined + ? { + stepName, + workflowName, + input: params.lazyStepInput, + ...ownershipStamp, + } + : { stepName, ...ownershipStamp }, + }, + startEventParams(fence) + ) ); if (!startResult.step) { From b5ea66588693475cea031264641c559c9f83b4df Mon Sep 17 00:00:00 2001 From: Peter Wielander Date: Fri, 31 Jul 2026 06:58:39 -0700 Subject: [PATCH 26/35] fix(world-local): order a slot-numbered event log by slot, not by start time `createdAt` is stamped when a write begins, before its final slot is known, so it disagrees with slot order in two ways: a writer that loses a slot re-proposes above the winner while keeping its earlier stamp, and a caller that reserves a range of slots for one flush commits them in whatever order the network returns. Replay consumes the log in list order and never sorts, so listing by `createdAt` hands it an order no execution produced. Slot events now report one shared order time and let the existing event-id tie-break do the ordering, matching the Postgres World's `orderBy(eventId)` and the Vercel World's sort key. ULID runs keep their wall-clock order, which their ids agree with anyway. --- .changeset/local-slot-order.md | 5 +++++ packages/world-local/src/fs.ts | 17 ++++++++++++---- .../world-local/src/storage/events-storage.ts | 20 +++++++++++++++++++ .../src/storage/slot-identity.test.ts | 11 ++++++++++ 4 files changed, 49 insertions(+), 4 deletions(-) create mode 100644 .changeset/local-slot-order.md diff --git a/.changeset/local-slot-order.md b/.changeset/local-slot-order.md new file mode 100644 index 0000000000..148f7df032 --- /dev/null +++ b/.changeset/local-slot-order.md @@ -0,0 +1,5 @@ +--- +'@workflow/world-local': patch +--- + +Read a position-numbered event log in position order, so a replay sees the log the order it was written diff --git a/packages/world-local/src/fs.ts b/packages/world-local/src/fs.ts index 5827f36896..bc370a2ca0 100644 --- a/packages/world-local/src/fs.ts +++ b/packages/world-local/src/fs.ts @@ -579,6 +579,14 @@ interface PaginatedFileSystemQueryConfig { cursor?: string; getCreatedAt(filename: string): Date | null; getId?(item: T): string; + /** + * The time an item sorts and paginates by, when that is not its `createdAt`. + * A slot-numbered event log orders by slot — the position is the order — and + * a writer that loses a slot re-proposes above the winner while keeping the + * stamp it started with, so `createdAt` there disagrees with the log. Such an + * item reports one shared time and lets the `getId` tie-break order it. + */ + getOrderTime?: (item: NoInfer) => number; } // Cursor format: "timestamp|id" for tie-breaking interface ParsedCursor { @@ -615,6 +623,7 @@ export async function paginatedFileSystemQuery( cursor, getCreatedAt, getId, + getOrderTime = (item: T) => item.createdAt.getTime(), } = config; // Validate filePrefix (typically `${runId}-`) so request-derived prefixes @@ -718,7 +727,7 @@ export async function paginatedFileSystemQuery( // Double-check cursor filtering with actual createdAt from JSON // (in case ULID timestamp differs from stored createdAt) if (parsedCursor) { - const itemTime = item.createdAt.getTime(); + const itemTime = getOrderTime(item); const cursorTime = parsedCursor.timestamp.getTime(); if (sortOrder === 'desc') { @@ -746,8 +755,8 @@ export async function paginatedFileSystemQuery( // 5. Sort by createdAt (and by ID for tie-breaking if getId is provided) validItems.sort((a, b) => { - const aTime = a.createdAt.getTime(); - const bTime = b.createdAt.getTime(); + const aTime = getOrderTime(a); + const bTime = getOrderTime(b); const timeComparison = sortOrder === 'asc' ? aTime - bTime : bTime - aTime; // If timestamps are equal and we have getId, use ID for stable sorting @@ -768,7 +777,7 @@ export async function paginatedFileSystemQuery( const nextCursor = items.length > 0 ? createCursor( - items[items.length - 1].createdAt, + new Date(getOrderTime(items[items.length - 1])), getId?.(items[items.length - 1]) ) : null; diff --git a/packages/world-local/src/storage/events-storage.ts b/packages/world-local/src/storage/events-storage.ts index 97376f5aab..0e6c0ce39d 100644 --- a/packages/world-local/src/storage/events-storage.ts +++ b/packages/world-local/src/storage/events-storage.ts @@ -131,6 +131,20 @@ function getMaxEventsPerRun(): number { // but a shared filesystem), exactly matching the cross-process // semantics without spawning subprocesses. +/** + * The time an event orders and paginates by. A slot-numbered run's order is its + * slot order — the position *is* the order, the way the sort key is for the + * other backends — so every such event reports the same time and lets the + * event-id tie-break do the ordering. Ordering those by `createdAt` reads the + * log in an order no replay produced: a writer that loses a slot re-proposes + * above the winner while keeping the stamp it started with, and a caller that + * reserves slots for a whole flush commits them in whatever order the network + * returns. A ULID-numbered run keeps its wall-clock order, which its ids agree + * with anyway. + */ +const eventOrderTime = (event: { eventId: string; createdAt: Date }): number => + slotFromId(event.eventId) === undefined ? event.createdAt.getTime() : 0; + const HookTokenClaimSchema = z.object({ // The token-claim writer below has always persisted `hookId`, but // this read schema previously omitted it, which is the bug fixed @@ -268,6 +282,7 @@ async function findExistingHookCreatedEventId( event.correlationId === correlationId, limit: 1, getCreatedAt: getObjectCreatedAt('evnt'), + getOrderTime: eventOrderTime, getId: (event) => event.eventId, }); return result.data[0]?.eventId ?? null; @@ -630,6 +645,7 @@ export function createEventsStorage( ? { cursor: params.sinceCursor } : {}), getCreatedAt: getObjectCreatedAt('evnt'), + getOrderTime: eventOrderTime, getId: (event) => event.eventId, }); const maxSlot = params?.maxSlot ?? 0; @@ -2821,6 +2837,7 @@ export function createEventsStorage( sortOrder: 'asc', limit: 1000, getCreatedAt: getObjectCreatedAt('evnt'), + getOrderTime: eventOrderTime, getId: (e) => e.eventId, }); events = allEvents.data; @@ -2869,6 +2886,7 @@ export function createEventsStorage( sortOrder: 'asc', cursor: params.sinceCursor, getCreatedAt: getObjectCreatedAt('evnt'), + getOrderTime: eventOrderTime, getId: (e) => e.eventId, }); events = @@ -2929,6 +2947,7 @@ export function createEventsStorage( limit: params.pagination?.limit, cursor: params.pagination?.cursor, getCreatedAt: getObjectCreatedAt('evnt'), + getOrderTime: eventOrderTime, getId: (event) => event.eventId, }); @@ -2961,6 +2980,7 @@ export function createEventsStorage( limit: params.pagination?.limit, cursor: params.pagination?.cursor, getCreatedAt: getObjectCreatedAt('evnt'), + getOrderTime: eventOrderTime, getId: (event) => event.eventId, }); diff --git a/packages/world-local/src/storage/slot-identity.test.ts b/packages/world-local/src/storage/slot-identity.test.ts index c21e215afa..5455e0a52f 100644 --- a/packages/world-local/src/storage/slot-identity.test.ts +++ b/packages/world-local/src/storage/slot-identity.test.ts @@ -120,6 +120,17 @@ describe('numbering', () => { await expect(slotsOf(runId)).resolves.toEqual([1, 2]); }); + it('lists the log in slot order, not in the order writes started', async () => { + // A writer that loses its slot re-proposes above the winner while keeping + // the wall-clock stamp it started with, so `createdAt` order and slot order + // disagree. Replay consumes the log in list order, so list order has to be + // slot order — what the sort key gives the other backends for free. + const runId = await newSlotRun(); + await createStep(runId, 'step_late', slotEventId(3)); + await createStep(runId, 'step_early', slotEventId(2)); + await expect(slotsOf(runId)).resolves.toEqual([1, 2, 3]); + }); + it('keeps a burst of concurrent writers dense', async () => { // The suspension flush issues every op at once. Density is what lets a // reader prove its log is complete, so a burst must not leave holes. From 6b2ccdb656201db50ea74f28b49d6660cef428ce Mon Sep 17 00:00:00 2001 From: Peter Wielander Date: Fri, 31 Jul 2026 06:58:39 -0700 Subject: [PATCH 27/35] fix(world-local): order a slot-numbered event log by slot, not by start time `createdAt` is stamped when a write begins, before its final slot is known, so it disagrees with slot order in two ways: a writer that loses a slot re-proposes above the winner while keeping its earlier stamp, and a caller that reserves a range of slots for one flush commits them in whatever order the network returns. Replay consumes the log in list order and never sorts, so listing by `createdAt` hands it an order no execution produced. Slot events now report one shared order time and let the existing event-id tie-break do the ordering, matching the Postgres World's `orderBy(eventId)` and the Vercel World's sort key. ULID runs keep their wall-clock order, which their ids agree with anyway. --- .changeset/local-slot-order.md | 5 +++++ packages/world-local/src/fs.ts | 17 ++++++++++++---- .../world-local/src/storage/events-storage.ts | 20 +++++++++++++++++++ .../src/storage/slot-identity.test.ts | 11 ++++++++++ 4 files changed, 49 insertions(+), 4 deletions(-) create mode 100644 .changeset/local-slot-order.md diff --git a/.changeset/local-slot-order.md b/.changeset/local-slot-order.md new file mode 100644 index 0000000000..148f7df032 --- /dev/null +++ b/.changeset/local-slot-order.md @@ -0,0 +1,5 @@ +--- +'@workflow/world-local': patch +--- + +Read a position-numbered event log in position order, so a replay sees the log the order it was written diff --git a/packages/world-local/src/fs.ts b/packages/world-local/src/fs.ts index aa3db9ee1e..8d2ec5d203 100644 --- a/packages/world-local/src/fs.ts +++ b/packages/world-local/src/fs.ts @@ -584,6 +584,14 @@ interface PaginatedFileSystemQueryConfig { cursor?: string; getCreatedAt(filename: string): Date | null; getId?(item: T): string; + /** + * The time an item sorts and paginates by, when that is not its `createdAt`. + * A slot-numbered event log orders by slot — the position is the order — and + * a writer that loses a slot re-proposes above the winner while keeping the + * stamp it started with, so `createdAt` there disagrees with the log. Such an + * item reports one shared time and lets the `getId` tie-break order it. + */ + getOrderTime?: (item: NoInfer) => number; } // Cursor format: "timestamp|id" for tie-breaking interface ParsedCursor { @@ -620,6 +628,7 @@ export async function paginatedFileSystemQuery( cursor, getCreatedAt, getId, + getOrderTime = (item: T) => item.createdAt.getTime(), } = config; // Validate filePrefix (typically `${runId}-`) so request-derived prefixes @@ -723,7 +732,7 @@ export async function paginatedFileSystemQuery( // Double-check cursor filtering with actual createdAt from JSON // (in case ULID timestamp differs from stored createdAt) if (parsedCursor) { - const itemTime = item.createdAt.getTime(); + const itemTime = getOrderTime(item); const cursorTime = parsedCursor.timestamp.getTime(); if (sortOrder === 'desc') { @@ -751,8 +760,8 @@ export async function paginatedFileSystemQuery( // 5. Sort by createdAt (and by ID for tie-breaking if getId is provided) validItems.sort((a, b) => { - const aTime = a.createdAt.getTime(); - const bTime = b.createdAt.getTime(); + const aTime = getOrderTime(a); + const bTime = getOrderTime(b); const timeComparison = sortOrder === 'asc' ? aTime - bTime : bTime - aTime; // If timestamps are equal and we have getId, use ID for stable sorting @@ -773,7 +782,7 @@ export async function paginatedFileSystemQuery( const nextCursor = items.length > 0 ? createCursor( - items[items.length - 1].createdAt, + new Date(getOrderTime(items[items.length - 1])), getId?.(items[items.length - 1]) ) : null; diff --git a/packages/world-local/src/storage/events-storage.ts b/packages/world-local/src/storage/events-storage.ts index 97376f5aab..0e6c0ce39d 100644 --- a/packages/world-local/src/storage/events-storage.ts +++ b/packages/world-local/src/storage/events-storage.ts @@ -131,6 +131,20 @@ function getMaxEventsPerRun(): number { // but a shared filesystem), exactly matching the cross-process // semantics without spawning subprocesses. +/** + * The time an event orders and paginates by. A slot-numbered run's order is its + * slot order — the position *is* the order, the way the sort key is for the + * other backends — so every such event reports the same time and lets the + * event-id tie-break do the ordering. Ordering those by `createdAt` reads the + * log in an order no replay produced: a writer that loses a slot re-proposes + * above the winner while keeping the stamp it started with, and a caller that + * reserves slots for a whole flush commits them in whatever order the network + * returns. A ULID-numbered run keeps its wall-clock order, which its ids agree + * with anyway. + */ +const eventOrderTime = (event: { eventId: string; createdAt: Date }): number => + slotFromId(event.eventId) === undefined ? event.createdAt.getTime() : 0; + const HookTokenClaimSchema = z.object({ // The token-claim writer below has always persisted `hookId`, but // this read schema previously omitted it, which is the bug fixed @@ -268,6 +282,7 @@ async function findExistingHookCreatedEventId( event.correlationId === correlationId, limit: 1, getCreatedAt: getObjectCreatedAt('evnt'), + getOrderTime: eventOrderTime, getId: (event) => event.eventId, }); return result.data[0]?.eventId ?? null; @@ -630,6 +645,7 @@ export function createEventsStorage( ? { cursor: params.sinceCursor } : {}), getCreatedAt: getObjectCreatedAt('evnt'), + getOrderTime: eventOrderTime, getId: (event) => event.eventId, }); const maxSlot = params?.maxSlot ?? 0; @@ -2821,6 +2837,7 @@ export function createEventsStorage( sortOrder: 'asc', limit: 1000, getCreatedAt: getObjectCreatedAt('evnt'), + getOrderTime: eventOrderTime, getId: (e) => e.eventId, }); events = allEvents.data; @@ -2869,6 +2886,7 @@ export function createEventsStorage( sortOrder: 'asc', cursor: params.sinceCursor, getCreatedAt: getObjectCreatedAt('evnt'), + getOrderTime: eventOrderTime, getId: (e) => e.eventId, }); events = @@ -2929,6 +2947,7 @@ export function createEventsStorage( limit: params.pagination?.limit, cursor: params.pagination?.cursor, getCreatedAt: getObjectCreatedAt('evnt'), + getOrderTime: eventOrderTime, getId: (event) => event.eventId, }); @@ -2961,6 +2980,7 @@ export function createEventsStorage( limit: params.pagination?.limit, cursor: params.pagination?.cursor, getCreatedAt: getObjectCreatedAt('evnt'), + getOrderTime: eventOrderTime, getId: (event) => event.eventId, }); diff --git a/packages/world-local/src/storage/slot-identity.test.ts b/packages/world-local/src/storage/slot-identity.test.ts index c21e215afa..5455e0a52f 100644 --- a/packages/world-local/src/storage/slot-identity.test.ts +++ b/packages/world-local/src/storage/slot-identity.test.ts @@ -120,6 +120,17 @@ describe('numbering', () => { await expect(slotsOf(runId)).resolves.toEqual([1, 2]); }); + it('lists the log in slot order, not in the order writes started', async () => { + // A writer that loses its slot re-proposes above the winner while keeping + // the wall-clock stamp it started with, so `createdAt` order and slot order + // disagree. Replay consumes the log in list order, so list order has to be + // slot order — what the sort key gives the other backends for free. + const runId = await newSlotRun(); + await createStep(runId, 'step_late', slotEventId(3)); + await createStep(runId, 'step_early', slotEventId(2)); + await expect(slotsOf(runId)).resolves.toEqual([1, 2, 3]); + }); + it('keeps a burst of concurrent writers dense', async () => { // The suspension flush issues every op at once. Density is what lets a // reader prove its log is complete, so a burst must not leave holes. From 41bc65ad0ae491a405dbef975be17f129fcb72b4 Mon Sep 17 00:00:00 2001 From: Peter Wielander Date: Fri, 31 Jul 2026 11:17:34 -0700 Subject: [PATCH 28/35] fix(core): stop rejecting healthy logs on late deliveries MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Two independent sources of `CorruptedEventLogError` on well-formed event logs, both found by dumping the logs of runs that failed that way. A hook delivery is ordered by when its event row commits, not by when the payload arrived, so a delivery that races a disposal — arriving first, committing second — lands after its own `hook_disposed` in the log. The hook's consumer retired on the disposal, so nothing consumed that event on any replay: a divergence that recurs identically every attempt and escalates to a terminal error. The consumer now stays registered as a tombstone and discards the late delivery, which is what `disposeHook` already assumed when it settled every awaiter. Separately, the unconsumed-event check gave the VM a flat 100ms of wall clock to register the next event's consumer. Real logs routinely need more: a hook payload fanning out into steps measures 254-717ms between the delivery and the first `step_created` it causes. The check now re-arms while a delivery is still in flight — the condition `scheduleWhenIdle` already polls — bounded by `WORKFLOW_DEFERRED_CHECK_MAX_GRACE_MS` so an abandoned delivery cannot park it forever. --- .changeset/late-hook-delivery-divergence.md | 6 + .../docs/v5/configuration/runtime-tuning.mdx | 6 + packages/core/src/events-consumer.test.ts | 69 +++++++++- packages/core/src/events-consumer.ts | 118 ++++++++++++++---- packages/core/src/private.ts | 15 ++- packages/core/src/workflow.ts | 13 +- packages/core/src/workflow/hook.test.ts | 66 +++++++++- packages/core/src/workflow/hook.ts | 41 +++++- 8 files changed, 292 insertions(+), 42 deletions(-) create mode 100644 .changeset/late-hook-delivery-divergence.md diff --git a/.changeset/late-hook-delivery-divergence.md b/.changeset/late-hook-delivery-divergence.md new file mode 100644 index 0000000000..ab62353ca8 --- /dev/null +++ b/.changeset/late-hook-delivery-divergence.md @@ -0,0 +1,6 @@ +--- +'@workflow/core': patch +'workflow': patch +--- + +Stop failing runs with a corrupted-event-log error when a hook delivery arrives after the hook was disposed, or when a step result is still being fetched diff --git a/docs/content/docs/v5/configuration/runtime-tuning.mdx b/docs/content/docs/v5/configuration/runtime-tuning.mdx index ba4a05b397..9569bb6f3d 100644 --- a/docs/content/docs/v5/configuration/runtime-tuning.mdx +++ b/docs/content/docs/v5/configuration/runtime-tuning.mdx @@ -198,6 +198,12 @@ These variables are primarily for tests, debugging, or unusual deployments. - Delay before the unconsumed-event check fires. - Minimum: `10`. +### `WORKFLOW_DEFERRED_CHECK_MAX_GRACE_MS` + +- Default: `15000` +- How long the unconsumed-event check keeps waiting while a step result or hook payload is still on its way to the workflow. An event whose consumer has not been registered yet looks exactly like an orphaned one, so the check waits rather than failing the run. +- Once this budget is spent the check reports regardless, so a delivery that never lands cannot keep a genuinely orphaned event from being detected. + ### `WORKFLOW_LOCK_POLL_INTERVAL_MS` - Default: `10` diff --git a/packages/core/src/events-consumer.test.ts b/packages/core/src/events-consumer.test.ts index ecf828f728..6294803363 100644 --- a/packages/core/src/events-consumer.test.ts +++ b/packages/core/src/events-consumer.test.ts @@ -1,7 +1,15 @@ import { withResolvers } from '@workflow/utils'; import type { Event } from '@workflow/world'; -import { describe, expect, it, vi } from 'vitest'; -import { EventConsumerResult, EventsConsumer } from './events-consumer.js'; +import { afterEach, describe, expect, it, vi } from 'vitest'; +import { + DEFERRED_CHECK_DELAY_MS, + EventConsumerResult, + EventsConsumer, +} from './events-consumer.js'; + +afterEach(() => { + vi.unstubAllEnvs(); +}); // Helper function to create mock events function createMockEvent(overrides: Partial = {}): Event { @@ -468,12 +476,63 @@ describe('EventsConsumer', () => { expect(consumer.eventIndex).toBe(1); }); - // Wait past the internal 100ms unconsumed-event setTimeout window to - // ensure the cancelled check truly does not fire. - await new Promise((resolve) => setTimeout(resolve, 150)); + // Wait past the internal unconsumed-event setTimeout window to ensure the + // cancelled check truly does not fire. + await new Promise((resolve) => + setTimeout(resolve, DEFERRED_CHECK_DELAY_MS * 1.5) + ); // The new callback consumed the event, so onUnconsumedEvent should NOT be called expect(onUnconsumedEvent).not.toHaveBeenCalled(); }); + + it('waits while a delivery is in flight, then reports once it lands', async () => { + vi.stubEnv('WORKFLOW_DEFERRED_CHECK_DELAY_MS', '10'); + const event = createMockEvent(); + const onUnconsumedEvent = vi.fn(); + let inFlight = true; + const consumer = new EventsConsumer([event], { + onUnconsumedEvent, + getPromiseQueue: () => Promise.resolve(), + isDeliveryInFlight: () => inFlight, + }); + + consumer.subscribe( + vi.fn().mockReturnValue(EventConsumerResult.NotConsumed) + ); + + // Many delay windows pass. A delivery still on its way to the workflow + // means the consumer for this event has not been registered YET — which + // is not the same thing as the event being orphaned. + await new Promise((resolve) => setTimeout(resolve, 200)); + expect(onUnconsumedEvent).not.toHaveBeenCalled(); + + inFlight = false; + await vi.waitFor(() => { + expect(onUnconsumedEvent).toHaveBeenCalledWith(event); + }); + }); + + it('reports once the grace budget runs out even if a delivery never lands', async () => { + vi.stubEnv('WORKFLOW_DEFERRED_CHECK_DELAY_MS', '10'); + vi.stubEnv('WORKFLOW_DEFERRED_CHECK_MAX_GRACE_MS', '50'); + const event = createMockEvent(); + const onUnconsumedEvent = vi.fn(); + const consumer = new EventsConsumer([event], { + onUnconsumedEvent, + getPromiseQueue: () => Promise.resolve(), + // Never clears: a delivery that is abandoned must not park the check + // forever, or a genuinely orphaned event would never be reported. + isDeliveryInFlight: () => true, + }); + + consumer.subscribe( + vi.fn().mockReturnValue(EventConsumerResult.NotConsumed) + ); + + await vi.waitFor(() => { + expect(onUnconsumedEvent).toHaveBeenCalledWith(event); + }); + }); }); }); diff --git a/packages/core/src/events-consumer.ts b/packages/core/src/events-consumer.ts index 4b7cf0742e..5db54b4169 100644 --- a/packages/core/src/events-consumer.ts +++ b/packages/core/src/events-consumer.ts @@ -26,6 +26,26 @@ const getDeferredCheckDelayMs = (): number => min: 10, }); +/** + * Upper bound on how long the unconsumed-event check keeps re-arming while a + * data delivery is still in flight (see `isDeliveryInFlight`). The delay above + * is a margin for a microtask chain; this is a margin for real async work — + * decrypting a hook payload, fetching a remote ref — that has to finish before + * the VM can resume the branch that registers the next event's consumer. + * + * Bounded rather than unbounded so a genuinely orphaned event still reports, + * and so a delivery that never lands cannot park the check forever. + */ +export const DEFERRED_CHECK_MAX_GRACE_MS = 15_000; + +/** Override: `WORKFLOW_DEFERRED_CHECK_MAX_GRACE_MS`. */ +const getDeferredCheckMaxGraceMs = (): number => + envNumber( + 'WORKFLOW_DEFERRED_CHECK_MAX_GRACE_MS', + DEFERRED_CHECK_MAX_GRACE_MS, + { integer: true, min: 0 } + ); + export enum EventConsumerResult { /** * Callback consumed the event, but should not be removed from the callbacks list @@ -65,6 +85,16 @@ export interface EventsConsumerOptions { * deserialization delays the resolve() that triggers the next subscribe(). */ getPromiseQueue: () => Promise; + /** + * Whether a data delivery (step result, hook payload) is still on its way to + * the workflow. The unconsumed-event check re-arms while this holds instead + * of reporting: an event whose consumer has not been registered yet is + * indistinguishable from an orphaned one by log inspection alone, and the + * promise-queue drain does not cover the gap between a delivery's `resolve()` + * and the VM body reaching its next `subscribe()`. Defaults to never in + * flight, which is the plain wall-clock behaviour. + */ + isDeliveryInFlight?: () => boolean; } export class EventsConsumer { @@ -74,6 +104,7 @@ export class EventsConsumer { private onConsumedEvent?: (event: Event) => void; private onUnconsumedEvent: (event: Event) => void; private getPromiseQueue: () => Promise; + private isDeliveryInFlight: () => boolean; private pendingUnconsumedCheck: Promise | null = null; private pendingUnconsumedTimeout: ReturnType | null = null; private unconsumedCheckVersion = 0; @@ -84,6 +115,7 @@ export class EventsConsumer { this.onConsumedEvent = options.onConsumedEvent; this.onUnconsumedEvent = options.onUnconsumedEvent; this.getPromiseQueue = options.getPromiseQueue; + this.isDeliveryInFlight = options.isDeliveryInFlight ?? (() => false); } /** @@ -200,32 +232,66 @@ export class EventsConsumer { // is still unconsumed after the queue drains, it's truly orphaned. if (currentEvent !== null) { const checkVersion = ++this.unconsumedCheckVersion; - this.pendingUnconsumedCheck = this.getPromiseQueue() - .then( - // Yield once after the first queue drain so promise chains resumed by - // that drain can run across the VM boundary and append any follow-up - // async work (for example: step_completed resolves -> for-await loop - // resumes -> the next hook payload starts hydrating). - () => new Promise((resolve) => setTimeout(resolve, 0)) - ) - .then(() => this.getPromiseQueue()) - .then(() => { - // Use a delayed setTimeout after the queue drains. The delay must be - // long enough for promise chains to propagate across the VM boundary - // (from resolve() in the host context through to the workflow code - // calling subscribe() in the VM context). Node.js does not guarantee - // that setTimeout(0) fires after all cross-context microtasks settle, - // so we use a small but non-zero delay. Any subscribe() call that - // arrives during this window will cancel the check via version - // invalidation + clearTimeout. - this.pendingUnconsumedTimeout = setTimeout(() => { - this.pendingUnconsumedTimeout = null; - if (this.unconsumedCheckVersion === checkVersion) { - this.pendingUnconsumedCheck = null; - this.onUnconsumedEvent(currentEvent); - } - }, getDeferredCheckDelayMs()); - }); + this.armUnconsumedCheck( + currentEvent, + checkVersion, + getDeferredCheckMaxGraceMs() + ); } } + + /** + * Wait for the promise queue to drain, then a short delay, then report + * `currentEvent` as unconsumed — unless a `subscribe()` invalidated + * `checkVersion` in the meantime, or a delivery is still in flight, in which + * case re-arm with `graceRemainingMs` reduced by the delay just spent. + */ + private armUnconsumedCheck( + currentEvent: Event, + checkVersion: number, + graceRemainingMs: number + ) { + const delay = getDeferredCheckDelayMs(); + this.pendingUnconsumedCheck = this.getPromiseQueue() + .then( + // Yield once after the first queue drain so promise chains resumed by + // that drain can run across the VM boundary and append any follow-up + // async work (for example: step_completed resolves -> for-await loop + // resumes -> the next hook payload starts hydrating). + () => new Promise((resolve) => setTimeout(resolve, 0)) + ) + .then(() => this.getPromiseQueue()) + .then(() => { + // Use a delayed setTimeout after the queue drains. The delay must be + // long enough for promise chains to propagate across the VM boundary + // (from resolve() in the host context through to the workflow code + // calling subscribe() in the VM context). Node.js does not guarantee + // that setTimeout(0) fires after all cross-context microtasks settle, + // so we use a small but non-zero delay. Any subscribe() call that + // arrives during this window will cancel the check via version + // invalidation + clearTimeout. + this.pendingUnconsumedTimeout = setTimeout(() => { + this.pendingUnconsumedTimeout = null; + if (this.unconsumedCheckVersion !== checkVersion) { + return; + } + if (graceRemainingMs > 0 && this.isDeliveryInFlight()) { + // A delivery is hydrating, or has resolved but is parked behind its + // deferral. The workflow body has not had the chance to register + // this event's consumer yet, so reporting now would reject a + // healthy run: the resulting `ReplayDivergenceError` recurs on + // every replay that is unlucky in the same way and escalates to a + // terminal `CorruptedEventLogError`. + this.armUnconsumedCheck( + currentEvent, + checkVersion, + graceRemainingMs - delay + ); + return; + } + this.pendingUnconsumedCheck = null; + this.onUnconsumedEvent(currentEvent); + }, delay); + }); + } } diff --git a/packages/core/src/private.ts b/packages/core/src/private.ts index a4498b3179..89b2e86a61 100644 --- a/packages/core/src/private.ts +++ b/packages/core/src/private.ts @@ -555,12 +555,25 @@ function hasParkedCommittedDelivery(ctx: WorkflowOrchestratorContext): boolean { * delivery still in flight. Empirically, replacing it with `queueMicrotask` * breaks hook/sleep `Promise.race` ordering (CorruptedEventLogError). */ +/** + * Whether some data delivery is still on its way to the workflow — the same + * two windows {@link scheduleWhenIdle} polls on, exposed for callers that need + * to test the condition without waiting on it. + * + * While this holds, the VM has not yet run the continuation that registers the + * next event's consumer, so "no consumer for this event" says nothing about + * whether the event log is well-formed. + */ +export function hasInFlightDelivery(ctx: WorkflowOrchestratorContext): boolean { + return ctx.pendingDeliveries > 0 || hasParkedCommittedDelivery(ctx); +} + export function scheduleWhenIdle( ctx: WorkflowOrchestratorContext, fn: () => void ): void { const check = () => { - if (ctx.pendingDeliveries > 0 || hasParkedCommittedDelivery(ctx)) { + if (hasInFlightDelivery(ctx)) { // A delivery is still hydrating, or is committed but parked behind its // deferral (whose resolve runs on a detached timer, not this queue). // Either way: let the queue drain, then re-check a timer tick later. diff --git a/packages/core/src/workflow.ts b/packages/core/src/workflow.ts index 97c9acee99..03f8d1e203 100644 --- a/packages/core/src/workflow.ts +++ b/packages/core/src/workflow.ts @@ -15,7 +15,10 @@ import { EventConsumerResult, EventsConsumer } from './events-consumer.js'; import type { QueueItem } from './global.js'; import { ENOTSUP, WorkflowSuspension } from './global.js'; import { runtimeLogger } from './logger.js'; -import type { WorkflowOrchestratorContext } from './private.js'; +import { + hasInFlightDelivery, + type WorkflowOrchestratorContext, +} from './private.js'; import { ReplayPayloadCache } from './replay-payload-cache.js'; import { getPortLazy } from './runtime/get-port-lazy.js'; import type { MutableEventLog } from './runtime/helpers.js'; @@ -208,6 +211,10 @@ export async function runWorkflow( // by step/hook/sleep callbacks as events are processed. const promiseQueueHolder = { current: Promise.resolve() }; + // Assigned immediately below. The consumer needs to test the context's + // delivery state, and the context needs the consumer. + let workflowContext: WorkflowOrchestratorContext; + const eventsConsumer = new EventsConsumer(events, { onConsumedEvent: (event) => { updateTimestamp(+event.createdAt); @@ -221,9 +228,11 @@ export async function runWorkflow( ); }, getPromiseQueue: () => promiseQueueHolder.current, + isDeliveryInFlight: () => + workflowContext !== undefined && hasInFlightDelivery(workflowContext), }); - const workflowContext: WorkflowOrchestratorContext = { + workflowContext = { runId: workflowRun.runId, encryptionKey, worldCapabilities, diff --git a/packages/core/src/workflow/hook.test.ts b/packages/core/src/workflow/hook.test.ts index 86cf8b54a3..fa913ff4e7 100644 --- a/packages/core/src/workflow/hook.test.ts +++ b/packages/core/src/workflow/hook.test.ts @@ -24,7 +24,10 @@ import { createWebhook } from './create-hook.js'; import { createCreateHook } from './hook.js'; // Helper to setup context to simulate a workflow run -function setupWorkflowContext(events: Event[]): WorkflowOrchestratorContext { +function setupWorkflowContext( + events: Event[], + onUnconsumedEvent: (event: Event) => void = () => {} +): WorkflowOrchestratorContext { const context = createContext({ seed: 'test', fixedTimestamp: 1753481739458, @@ -43,7 +46,7 @@ function setupWorkflowContext(events: Event[]): WorkflowOrchestratorContext { replayPayloadCache: new ReplayPayloadCache(undefined), globalThis: context.globalThis, eventsConsumer: new EventsConsumer(events, { - onUnconsumedEvent: () => {}, + onUnconsumedEvent, getPromiseQueue: () => Promise.resolve(), }), invocationsQueue: new Map(), @@ -471,6 +474,65 @@ describe('createCreateHook', () => { expect(runtimeErrors).toHaveLength(0); }); + it('should discard a hook_received ordered after the hook_disposed', async () => { + // The world orders a delivery by when its event row commits, not by when + // the payload arrived, so a delivery that raced the disposal can land after + // it in the log. Nothing else in the run can consume that event, so the + // hook's own consumer has to swallow it — otherwise the events consumer + // reports an orphan and the replay diverges on a well-formed log. + const ops: Promise[] = []; + const onUnconsumedEvent = vi.fn(); + const ctx = setupWorkflowContext( + [ + { + eventId: 'evnt_0', + runId: 'wrun_123', + eventType: 'hook_created', + correlationId: 'hook_01K11TFZ62YS0YYFDQ3E8B9YCV', + eventData: { token: 'test-token' }, + createdAt: new Date(), + }, + { + eventId: 'evnt_1', + runId: 'wrun_123', + eventType: 'hook_disposed', + correlationId: 'hook_01K11TFZ62YS0YYFDQ3E8B9YCV', + eventData: { token: 'test-token' }, + createdAt: new Date(), + }, + { + eventId: 'evnt_2', + runId: 'wrun_123', + eventType: 'hook_received', + correlationId: 'hook_01K11TFZ62YS0YYFDQ3E8B9YCV', + eventData: { + token: 'test-token', + payload: await dehydrateStepReturnValue( + { message: 'lost the race' }, + 'wrun_test', + undefined, + ops + ), + }, + createdAt: new Date(), + }, + ], + onUnconsumedEvent + ); + + const createHook = createCreateHook(ctx); + createHook({ token: 'test-token' }); + + // The whole log is consumed: the disposal retires the hook and the late + // delivery is dropped on the floor. + await vi.waitFor(() => { + expect(ctx.eventsConsumer.eventIndex).toBe(3); + }); + expect(onUnconsumedEvent).not.toHaveBeenCalled(); + expect(ctx.onWorkflowError).not.toHaveBeenCalled(); + expect(ctx.invocationsQueue.size).toBe(0); + }); + it('should handle multiple hook_received events with iterator', async () => { const ops: Promise[] = []; const ctx = setupWorkflowContext([ diff --git a/packages/core/src/workflow/hook.ts b/packages/core/src/workflow/hook.ts index e079827051..f7d1f209ac 100644 --- a/packages/core/src/workflow/hook.ts +++ b/packages/core/src/workflow/hook.ts @@ -154,8 +154,9 @@ export function createCreateHook(ctx: WorkflowOrchestratorContext) { eventLogEmpty = true; if ( - (promises.length > 0 && payloadsQueue.length === 0) || - (getConflictPromises.length > 0 && !hasCreated && !hasConflict) + !hasDisposedEvent && + ((promises.length > 0 && payloadsQueue.length === 0) || + (getConflictPromises.length > 0 && !hasCreated && !hasConflict)) ) { scheduleWhenIdle(ctx, () => { ctx.onWorkflowError( @@ -171,6 +172,31 @@ export function createCreateHook(ctx: WorkflowOrchestratorContext) { return EventConsumerResult.NotConsumed; } + if (hasDisposedEvent) { + // A delivery ordered AFTER this hook's own `hook_disposed`. The world + // orders a delivery by when its event row is written, not by when the + // payload arrived, so a delivery that raced the disposal — arriving + // first, committing second — lands here. Swallow it: the hook is gone, + // there is no consumer to hand the payload to, and every awaiter was + // already settled by `disposeHook`. + // + // This consumer must stay registered to do that. Retiring it on + // `hook_disposed` leaves the late delivery with no consumer at all, + // which the events consumer reports as an orphaned event — + // a `ReplayDivergenceError` that recurs on every replay of a log that + // is otherwise perfectly well-formed, escalating to a terminal + // `CorruptedEventLogError`. + webhookLogger.warn( + 'Discarding a hook delivery ordered after disposal', + { + correlationId, + eventId: event.eventId, + eventType: event.eventType, + } + ); + return EventConsumerResult.Consumed; + } + const eventToken = 'eventData' in event && event.eventData && 'token' in event.eventData ? event.eventData.token @@ -418,8 +444,10 @@ export function createCreateHook(ctx: WorkflowOrchestratorContext) { ctx.invocationsQueue.delete(correlationId); // Mark that the event log confirms disposal happened hasDisposedEvent = true; - // We're done processing any more events for this hook - return EventConsumerResult.Finished; + // Stay registered as a tombstone rather than retiring: a delivery that + // raced this disposal can still be ordered after it, and nothing else + // in the run can consume it. See the `hasDisposedEvent` branch above. + return EventConsumerResult.Consumed; } // This replay installed a different consumer than the stored event needs. @@ -539,8 +567,9 @@ export function createCreateHook(ctx: WorkflowOrchestratorContext) { // Drain any pending promises that are waiting for payloads. // Without this, promises created by `await hook` or the async iterator's - // `yield await this` would hang forever since the event consumer will - // never deliver another hook_received after disposal. + // `yield await this` would hang forever: a hook_received ordered after + // the disposal is discarded rather than handed to an awaiter, so nothing + // will ever settle them. if (promises.length > 0) { promises.length = 0; scheduleWhenIdle(ctx, () => { From b67828c212a97f8e88b078675671fa2fb2aecd31 Mon Sep 17 00:00:00 2001 From: Peter Wielander Date: Fri, 31 Jul 2026 11:17:34 -0700 Subject: [PATCH 29/35] fix(core): stop rejecting healthy logs on late deliveries MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Two independent sources of `CorruptedEventLogError` on well-formed event logs, both found by dumping the logs of runs that failed that way. A hook delivery is ordered by when its event row commits, not by when the payload arrived, so a delivery that races a disposal — arriving first, committing second — lands after its own `hook_disposed` in the log. The hook's consumer retired on the disposal, so nothing consumed that event on any replay: a divergence that recurs identically every attempt and escalates to a terminal error. The consumer now stays registered as a tombstone and discards the late delivery, which is what `disposeHook` already assumed when it settled every awaiter. Separately, the unconsumed-event check gave the VM a flat 100ms of wall clock to register the next event's consumer. Real logs routinely need more: a hook payload fanning out into steps measures 254-717ms between the delivery and the first `step_created` it causes. The check now re-arms while a delivery is still in flight — the condition `scheduleWhenIdle` already polls — bounded by `WORKFLOW_DEFERRED_CHECK_MAX_GRACE_MS` so an abandoned delivery cannot park it forever. --- .changeset/late-hook-delivery-divergence.md | 6 + .../docs/v5/configuration/runtime-tuning.mdx | 6 + packages/core/src/events-consumer.test.ts | 69 +++++++++- packages/core/src/events-consumer.ts | 118 ++++++++++++++---- packages/core/src/private.ts | 15 ++- packages/core/src/workflow.ts | 13 +- packages/core/src/workflow/hook.test.ts | 66 +++++++++- packages/core/src/workflow/hook.ts | 41 +++++- 8 files changed, 292 insertions(+), 42 deletions(-) create mode 100644 .changeset/late-hook-delivery-divergence.md diff --git a/.changeset/late-hook-delivery-divergence.md b/.changeset/late-hook-delivery-divergence.md new file mode 100644 index 0000000000..ab62353ca8 --- /dev/null +++ b/.changeset/late-hook-delivery-divergence.md @@ -0,0 +1,6 @@ +--- +'@workflow/core': patch +'workflow': patch +--- + +Stop failing runs with a corrupted-event-log error when a hook delivery arrives after the hook was disposed, or when a step result is still being fetched diff --git a/docs/content/docs/v5/configuration/runtime-tuning.mdx b/docs/content/docs/v5/configuration/runtime-tuning.mdx index ba4a05b397..9569bb6f3d 100644 --- a/docs/content/docs/v5/configuration/runtime-tuning.mdx +++ b/docs/content/docs/v5/configuration/runtime-tuning.mdx @@ -198,6 +198,12 @@ These variables are primarily for tests, debugging, or unusual deployments. - Delay before the unconsumed-event check fires. - Minimum: `10`. +### `WORKFLOW_DEFERRED_CHECK_MAX_GRACE_MS` + +- Default: `15000` +- How long the unconsumed-event check keeps waiting while a step result or hook payload is still on its way to the workflow. An event whose consumer has not been registered yet looks exactly like an orphaned one, so the check waits rather than failing the run. +- Once this budget is spent the check reports regardless, so a delivery that never lands cannot keep a genuinely orphaned event from being detected. + ### `WORKFLOW_LOCK_POLL_INTERVAL_MS` - Default: `10` diff --git a/packages/core/src/events-consumer.test.ts b/packages/core/src/events-consumer.test.ts index ecf828f728..6294803363 100644 --- a/packages/core/src/events-consumer.test.ts +++ b/packages/core/src/events-consumer.test.ts @@ -1,7 +1,15 @@ import { withResolvers } from '@workflow/utils'; import type { Event } from '@workflow/world'; -import { describe, expect, it, vi } from 'vitest'; -import { EventConsumerResult, EventsConsumer } from './events-consumer.js'; +import { afterEach, describe, expect, it, vi } from 'vitest'; +import { + DEFERRED_CHECK_DELAY_MS, + EventConsumerResult, + EventsConsumer, +} from './events-consumer.js'; + +afterEach(() => { + vi.unstubAllEnvs(); +}); // Helper function to create mock events function createMockEvent(overrides: Partial = {}): Event { @@ -468,12 +476,63 @@ describe('EventsConsumer', () => { expect(consumer.eventIndex).toBe(1); }); - // Wait past the internal 100ms unconsumed-event setTimeout window to - // ensure the cancelled check truly does not fire. - await new Promise((resolve) => setTimeout(resolve, 150)); + // Wait past the internal unconsumed-event setTimeout window to ensure the + // cancelled check truly does not fire. + await new Promise((resolve) => + setTimeout(resolve, DEFERRED_CHECK_DELAY_MS * 1.5) + ); // The new callback consumed the event, so onUnconsumedEvent should NOT be called expect(onUnconsumedEvent).not.toHaveBeenCalled(); }); + + it('waits while a delivery is in flight, then reports once it lands', async () => { + vi.stubEnv('WORKFLOW_DEFERRED_CHECK_DELAY_MS', '10'); + const event = createMockEvent(); + const onUnconsumedEvent = vi.fn(); + let inFlight = true; + const consumer = new EventsConsumer([event], { + onUnconsumedEvent, + getPromiseQueue: () => Promise.resolve(), + isDeliveryInFlight: () => inFlight, + }); + + consumer.subscribe( + vi.fn().mockReturnValue(EventConsumerResult.NotConsumed) + ); + + // Many delay windows pass. A delivery still on its way to the workflow + // means the consumer for this event has not been registered YET — which + // is not the same thing as the event being orphaned. + await new Promise((resolve) => setTimeout(resolve, 200)); + expect(onUnconsumedEvent).not.toHaveBeenCalled(); + + inFlight = false; + await vi.waitFor(() => { + expect(onUnconsumedEvent).toHaveBeenCalledWith(event); + }); + }); + + it('reports once the grace budget runs out even if a delivery never lands', async () => { + vi.stubEnv('WORKFLOW_DEFERRED_CHECK_DELAY_MS', '10'); + vi.stubEnv('WORKFLOW_DEFERRED_CHECK_MAX_GRACE_MS', '50'); + const event = createMockEvent(); + const onUnconsumedEvent = vi.fn(); + const consumer = new EventsConsumer([event], { + onUnconsumedEvent, + getPromiseQueue: () => Promise.resolve(), + // Never clears: a delivery that is abandoned must not park the check + // forever, or a genuinely orphaned event would never be reported. + isDeliveryInFlight: () => true, + }); + + consumer.subscribe( + vi.fn().mockReturnValue(EventConsumerResult.NotConsumed) + ); + + await vi.waitFor(() => { + expect(onUnconsumedEvent).toHaveBeenCalledWith(event); + }); + }); }); }); diff --git a/packages/core/src/events-consumer.ts b/packages/core/src/events-consumer.ts index 4b7cf0742e..5db54b4169 100644 --- a/packages/core/src/events-consumer.ts +++ b/packages/core/src/events-consumer.ts @@ -26,6 +26,26 @@ const getDeferredCheckDelayMs = (): number => min: 10, }); +/** + * Upper bound on how long the unconsumed-event check keeps re-arming while a + * data delivery is still in flight (see `isDeliveryInFlight`). The delay above + * is a margin for a microtask chain; this is a margin for real async work — + * decrypting a hook payload, fetching a remote ref — that has to finish before + * the VM can resume the branch that registers the next event's consumer. + * + * Bounded rather than unbounded so a genuinely orphaned event still reports, + * and so a delivery that never lands cannot park the check forever. + */ +export const DEFERRED_CHECK_MAX_GRACE_MS = 15_000; + +/** Override: `WORKFLOW_DEFERRED_CHECK_MAX_GRACE_MS`. */ +const getDeferredCheckMaxGraceMs = (): number => + envNumber( + 'WORKFLOW_DEFERRED_CHECK_MAX_GRACE_MS', + DEFERRED_CHECK_MAX_GRACE_MS, + { integer: true, min: 0 } + ); + export enum EventConsumerResult { /** * Callback consumed the event, but should not be removed from the callbacks list @@ -65,6 +85,16 @@ export interface EventsConsumerOptions { * deserialization delays the resolve() that triggers the next subscribe(). */ getPromiseQueue: () => Promise; + /** + * Whether a data delivery (step result, hook payload) is still on its way to + * the workflow. The unconsumed-event check re-arms while this holds instead + * of reporting: an event whose consumer has not been registered yet is + * indistinguishable from an orphaned one by log inspection alone, and the + * promise-queue drain does not cover the gap between a delivery's `resolve()` + * and the VM body reaching its next `subscribe()`. Defaults to never in + * flight, which is the plain wall-clock behaviour. + */ + isDeliveryInFlight?: () => boolean; } export class EventsConsumer { @@ -74,6 +104,7 @@ export class EventsConsumer { private onConsumedEvent?: (event: Event) => void; private onUnconsumedEvent: (event: Event) => void; private getPromiseQueue: () => Promise; + private isDeliveryInFlight: () => boolean; private pendingUnconsumedCheck: Promise | null = null; private pendingUnconsumedTimeout: ReturnType | null = null; private unconsumedCheckVersion = 0; @@ -84,6 +115,7 @@ export class EventsConsumer { this.onConsumedEvent = options.onConsumedEvent; this.onUnconsumedEvent = options.onUnconsumedEvent; this.getPromiseQueue = options.getPromiseQueue; + this.isDeliveryInFlight = options.isDeliveryInFlight ?? (() => false); } /** @@ -200,32 +232,66 @@ export class EventsConsumer { // is still unconsumed after the queue drains, it's truly orphaned. if (currentEvent !== null) { const checkVersion = ++this.unconsumedCheckVersion; - this.pendingUnconsumedCheck = this.getPromiseQueue() - .then( - // Yield once after the first queue drain so promise chains resumed by - // that drain can run across the VM boundary and append any follow-up - // async work (for example: step_completed resolves -> for-await loop - // resumes -> the next hook payload starts hydrating). - () => new Promise((resolve) => setTimeout(resolve, 0)) - ) - .then(() => this.getPromiseQueue()) - .then(() => { - // Use a delayed setTimeout after the queue drains. The delay must be - // long enough for promise chains to propagate across the VM boundary - // (from resolve() in the host context through to the workflow code - // calling subscribe() in the VM context). Node.js does not guarantee - // that setTimeout(0) fires after all cross-context microtasks settle, - // so we use a small but non-zero delay. Any subscribe() call that - // arrives during this window will cancel the check via version - // invalidation + clearTimeout. - this.pendingUnconsumedTimeout = setTimeout(() => { - this.pendingUnconsumedTimeout = null; - if (this.unconsumedCheckVersion === checkVersion) { - this.pendingUnconsumedCheck = null; - this.onUnconsumedEvent(currentEvent); - } - }, getDeferredCheckDelayMs()); - }); + this.armUnconsumedCheck( + currentEvent, + checkVersion, + getDeferredCheckMaxGraceMs() + ); } } + + /** + * Wait for the promise queue to drain, then a short delay, then report + * `currentEvent` as unconsumed — unless a `subscribe()` invalidated + * `checkVersion` in the meantime, or a delivery is still in flight, in which + * case re-arm with `graceRemainingMs` reduced by the delay just spent. + */ + private armUnconsumedCheck( + currentEvent: Event, + checkVersion: number, + graceRemainingMs: number + ) { + const delay = getDeferredCheckDelayMs(); + this.pendingUnconsumedCheck = this.getPromiseQueue() + .then( + // Yield once after the first queue drain so promise chains resumed by + // that drain can run across the VM boundary and append any follow-up + // async work (for example: step_completed resolves -> for-await loop + // resumes -> the next hook payload starts hydrating). + () => new Promise((resolve) => setTimeout(resolve, 0)) + ) + .then(() => this.getPromiseQueue()) + .then(() => { + // Use a delayed setTimeout after the queue drains. The delay must be + // long enough for promise chains to propagate across the VM boundary + // (from resolve() in the host context through to the workflow code + // calling subscribe() in the VM context). Node.js does not guarantee + // that setTimeout(0) fires after all cross-context microtasks settle, + // so we use a small but non-zero delay. Any subscribe() call that + // arrives during this window will cancel the check via version + // invalidation + clearTimeout. + this.pendingUnconsumedTimeout = setTimeout(() => { + this.pendingUnconsumedTimeout = null; + if (this.unconsumedCheckVersion !== checkVersion) { + return; + } + if (graceRemainingMs > 0 && this.isDeliveryInFlight()) { + // A delivery is hydrating, or has resolved but is parked behind its + // deferral. The workflow body has not had the chance to register + // this event's consumer yet, so reporting now would reject a + // healthy run: the resulting `ReplayDivergenceError` recurs on + // every replay that is unlucky in the same way and escalates to a + // terminal `CorruptedEventLogError`. + this.armUnconsumedCheck( + currentEvent, + checkVersion, + graceRemainingMs - delay + ); + return; + } + this.pendingUnconsumedCheck = null; + this.onUnconsumedEvent(currentEvent); + }, delay); + }); + } } diff --git a/packages/core/src/private.ts b/packages/core/src/private.ts index a4498b3179..89b2e86a61 100644 --- a/packages/core/src/private.ts +++ b/packages/core/src/private.ts @@ -555,12 +555,25 @@ function hasParkedCommittedDelivery(ctx: WorkflowOrchestratorContext): boolean { * delivery still in flight. Empirically, replacing it with `queueMicrotask` * breaks hook/sleep `Promise.race` ordering (CorruptedEventLogError). */ +/** + * Whether some data delivery is still on its way to the workflow — the same + * two windows {@link scheduleWhenIdle} polls on, exposed for callers that need + * to test the condition without waiting on it. + * + * While this holds, the VM has not yet run the continuation that registers the + * next event's consumer, so "no consumer for this event" says nothing about + * whether the event log is well-formed. + */ +export function hasInFlightDelivery(ctx: WorkflowOrchestratorContext): boolean { + return ctx.pendingDeliveries > 0 || hasParkedCommittedDelivery(ctx); +} + export function scheduleWhenIdle( ctx: WorkflowOrchestratorContext, fn: () => void ): void { const check = () => { - if (ctx.pendingDeliveries > 0 || hasParkedCommittedDelivery(ctx)) { + if (hasInFlightDelivery(ctx)) { // A delivery is still hydrating, or is committed but parked behind its // deferral (whose resolve runs on a detached timer, not this queue). // Either way: let the queue drain, then re-check a timer tick later. diff --git a/packages/core/src/workflow.ts b/packages/core/src/workflow.ts index 9fb568d955..76fd5e00dd 100644 --- a/packages/core/src/workflow.ts +++ b/packages/core/src/workflow.ts @@ -15,7 +15,10 @@ import { EventConsumerResult, EventsConsumer } from './events-consumer.js'; import type { QueueItem } from './global.js'; import { ENOTSUP, WorkflowSuspension } from './global.js'; import { runtimeLogger } from './logger.js'; -import type { WorkflowOrchestratorContext } from './private.js'; +import { + hasInFlightDelivery, + type WorkflowOrchestratorContext, +} from './private.js'; import { ReplayPayloadCache } from './replay-payload-cache.js'; import { getPortLazy } from './runtime/get-port-lazy.js'; import type { MutableEventLog } from './runtime/helpers.js'; @@ -208,6 +211,10 @@ export async function runWorkflow( // by step/hook/sleep callbacks as events are processed. const promiseQueueHolder = { current: Promise.resolve() }; + // Assigned immediately below. The consumer needs to test the context's + // delivery state, and the context needs the consumer. + let workflowContext: WorkflowOrchestratorContext; + const eventsConsumer = new EventsConsumer(events, { onConsumedEvent: (event) => { updateTimestamp(+event.createdAt); @@ -227,9 +234,11 @@ export async function runWorkflow( ); }, getPromiseQueue: () => promiseQueueHolder.current, + isDeliveryInFlight: () => + workflowContext !== undefined && hasInFlightDelivery(workflowContext), }); - const workflowContext: WorkflowOrchestratorContext = { + workflowContext = { runId: workflowRun.runId, encryptionKey, worldCapabilities, diff --git a/packages/core/src/workflow/hook.test.ts b/packages/core/src/workflow/hook.test.ts index 4bda06998a..307293f571 100644 --- a/packages/core/src/workflow/hook.test.ts +++ b/packages/core/src/workflow/hook.test.ts @@ -24,7 +24,10 @@ import { createWebhook } from './create-hook.js'; import { createCreateHook } from './hook.js'; // Helper to setup context to simulate a workflow run -function setupWorkflowContext(events: Event[]): WorkflowOrchestratorContext { +function setupWorkflowContext( + events: Event[], + onUnconsumedEvent: (event: Event) => void = () => {} +): WorkflowOrchestratorContext { const context = createContext({ seed: 'test', fixedTimestamp: 1753481739458, @@ -43,7 +46,7 @@ function setupWorkflowContext(events: Event[]): WorkflowOrchestratorContext { replayPayloadCache: new ReplayPayloadCache(undefined), globalThis: context.globalThis, eventsConsumer: new EventsConsumer(events, { - onUnconsumedEvent: () => {}, + onUnconsumedEvent, getPromiseQueue: () => Promise.resolve(), }), invocationsQueue: new Map(), @@ -471,6 +474,65 @@ describe('createCreateHook', () => { expect(runtimeErrors).toHaveLength(0); }); + it('should discard a hook_received ordered after the hook_disposed', async () => { + // The world orders a delivery by when its event row commits, not by when + // the payload arrived, so a delivery that raced the disposal can land after + // it in the log. Nothing else in the run can consume that event, so the + // hook's own consumer has to swallow it — otherwise the events consumer + // reports an orphan and the replay diverges on a well-formed log. + const ops: Promise[] = []; + const onUnconsumedEvent = vi.fn(); + const ctx = setupWorkflowContext( + [ + { + eventId: 'evnt_0', + runId: 'wrun_123', + eventType: 'hook_created', + correlationId: 'hook_01K11TFZ62YS0YYFDQ3E8B9YCV', + eventData: { token: 'test-token' }, + createdAt: new Date(), + }, + { + eventId: 'evnt_1', + runId: 'wrun_123', + eventType: 'hook_disposed', + correlationId: 'hook_01K11TFZ62YS0YYFDQ3E8B9YCV', + eventData: { token: 'test-token' }, + createdAt: new Date(), + }, + { + eventId: 'evnt_2', + runId: 'wrun_123', + eventType: 'hook_received', + correlationId: 'hook_01K11TFZ62YS0YYFDQ3E8B9YCV', + eventData: { + token: 'test-token', + payload: await dehydrateStepReturnValue( + { message: 'lost the race' }, + 'wrun_test', + undefined, + ops + ), + }, + createdAt: new Date(), + }, + ], + onUnconsumedEvent + ); + + const createHook = createCreateHook(ctx); + createHook({ token: 'test-token' }); + + // The whole log is consumed: the disposal retires the hook and the late + // delivery is dropped on the floor. + await vi.waitFor(() => { + expect(ctx.eventsConsumer.eventIndex).toBe(3); + }); + expect(onUnconsumedEvent).not.toHaveBeenCalled(); + expect(ctx.onWorkflowError).not.toHaveBeenCalled(); + expect(ctx.invocationsQueue.size).toBe(0); + }); + it('should handle multiple hook_received events with iterator', async () => { const ops: Promise[] = []; const ctx = setupWorkflowContext([ diff --git a/packages/core/src/workflow/hook.ts b/packages/core/src/workflow/hook.ts index e079827051..f7d1f209ac 100644 --- a/packages/core/src/workflow/hook.ts +++ b/packages/core/src/workflow/hook.ts @@ -154,8 +154,9 @@ export function createCreateHook(ctx: WorkflowOrchestratorContext) { eventLogEmpty = true; if ( - (promises.length > 0 && payloadsQueue.length === 0) || - (getConflictPromises.length > 0 && !hasCreated && !hasConflict) + !hasDisposedEvent && + ((promises.length > 0 && payloadsQueue.length === 0) || + (getConflictPromises.length > 0 && !hasCreated && !hasConflict)) ) { scheduleWhenIdle(ctx, () => { ctx.onWorkflowError( @@ -171,6 +172,31 @@ export function createCreateHook(ctx: WorkflowOrchestratorContext) { return EventConsumerResult.NotConsumed; } + if (hasDisposedEvent) { + // A delivery ordered AFTER this hook's own `hook_disposed`. The world + // orders a delivery by when its event row is written, not by when the + // payload arrived, so a delivery that raced the disposal — arriving + // first, committing second — lands here. Swallow it: the hook is gone, + // there is no consumer to hand the payload to, and every awaiter was + // already settled by `disposeHook`. + // + // This consumer must stay registered to do that. Retiring it on + // `hook_disposed` leaves the late delivery with no consumer at all, + // which the events consumer reports as an orphaned event — + // a `ReplayDivergenceError` that recurs on every replay of a log that + // is otherwise perfectly well-formed, escalating to a terminal + // `CorruptedEventLogError`. + webhookLogger.warn( + 'Discarding a hook delivery ordered after disposal', + { + correlationId, + eventId: event.eventId, + eventType: event.eventType, + } + ); + return EventConsumerResult.Consumed; + } + const eventToken = 'eventData' in event && event.eventData && 'token' in event.eventData ? event.eventData.token @@ -418,8 +444,10 @@ export function createCreateHook(ctx: WorkflowOrchestratorContext) { ctx.invocationsQueue.delete(correlationId); // Mark that the event log confirms disposal happened hasDisposedEvent = true; - // We're done processing any more events for this hook - return EventConsumerResult.Finished; + // Stay registered as a tombstone rather than retiring: a delivery that + // raced this disposal can still be ordered after it, and nothing else + // in the run can consume it. See the `hasDisposedEvent` branch above. + return EventConsumerResult.Consumed; } // This replay installed a different consumer than the stored event needs. @@ -539,8 +567,9 @@ export function createCreateHook(ctx: WorkflowOrchestratorContext) { // Drain any pending promises that are waiting for payloads. // Without this, promises created by `await hook` or the async iterator's - // `yield await this` would hang forever since the event consumer will - // never deliver another hook_received after disposal. + // `yield await this` would hang forever: a hook_received ordered after + // the disposal is discarded rather than handed to an awaiter, so nothing + // will ever settle them. if (promises.length > 0) { promises.length = 0; scheduleWhenIdle(ctx, () => { From a02b180881053aa95de4c0e54391a5fb07f0c8a0 Mon Sep 17 00:00:00 2001 From: Peter Wielander Date: Fri, 31 Jul 2026 11:27:13 -0700 Subject: [PATCH 30/35] docs(world): stop calling a dense log a completeness proof Contiguous allocation is not the same thing as a gap-free published log: a slot claimed by an operation that then fails for a reason of its own is never filled, and once a later slot is published that gap is permanent. What the scheme actually buys is explicit contention and a log that reads in write order. Nothing consumed the proof, so this is a comment and docs correction. --- .../docs/v5/configuration/runtime-tuning.mdx | 2 +- packages/world-postgres/src/slots.ts | 9 +++++---- .../world-postgres/test/slot-identity.test.ts | 6 +++--- packages/world-vercel/src/events-v4.test.ts | 2 +- packages/world/src/slot-identity.ts | 16 +++++++++++----- 5 files changed, 21 insertions(+), 14 deletions(-) diff --git a/docs/content/docs/v5/configuration/runtime-tuning.mdx b/docs/content/docs/v5/configuration/runtime-tuning.mdx index 9569bb6f3d..254b0d4815 100644 --- a/docs/content/docs/v5/configuration/runtime-tuning.mdx +++ b/docs/content/docs/v5/configuration/runtime-tuning.mdx @@ -51,7 +51,7 @@ For example, a workflow can run a 10-minute inline step even with `WORKFLOW_REPL ### `WORKFLOW_SLOT_IDENTITY` - Default: enabled -- Numbers a new run's events by position instead of by ULID: `evnt_…001` is the run's first event, `evnt_…002` its second, with no gaps. A reader can then prove its copy of the log is complete, because the highest number is the event count. +- Numbers a new run's events by position instead of by ULID: `evnt_…001` is the run's first event, `evnt_…002` its second. Positions are allocated in order, so the log reads in the order it was written regardless of clock skew between writers. - Contention becomes explicit rather than silent. Two writers proposing the same position cannot both win: the loser gets a 409 ([`SlotConflictError`](/docs/api-reference/workflow-errors/slot-conflict-error)) carrying the events it was missing, and the runtime merges them, replays, and re-proposes at the next free position. - Applies only to runs created while it is enabled. A run keeps the identity scheme it was created with for life, so turning the flag on or off never affects runs already in flight. - Requires a World that supports it. A World that does not rejects the run outright rather than mis-numbering its events. diff --git a/packages/world-postgres/src/slots.ts b/packages/world-postgres/src/slots.ts index cd0a54f6f5..6a0366a636 100644 --- a/packages/world-postgres/src/slots.ts +++ b/packages/world-postgres/src/slots.ts @@ -2,10 +2,11 @@ * Slot identity for the postgres world. * * A slot-numbered run names its events by position: `evnt_…001` is the first - * event of the run, `evnt_…002` the second, with no gaps. Density is the point - * — it is what lets a reader prove its copy of a log is complete — so the two - * things this module has to get right are that a position is written at most - * once and that a lost position is never left behind as a hole. + * event of the run, `evnt_…002` the second. Contention on a position is the + * point — it is what makes a concurrent write detectable rather than silent — + * so the two things this module has to get right are that a position is written + * at most once and that a position this allocator loses is retried rather than + * abandoned as a hole. * * The authority for both is the events table's primary key, `(run_id, id)`: the * INSERT either lands or raises a unique violation, and a writer that loses the diff --git a/packages/world-postgres/test/slot-identity.test.ts b/packages/world-postgres/test/slot-identity.test.ts index 8e3f04c8a4..a6cf498651 100644 --- a/packages/world-postgres/test/slot-identity.test.ts +++ b/packages/world-postgres/test/slot-identity.test.ts @@ -259,8 +259,8 @@ describe('Slot identity (Postgres integration)', () => { describe('contention', () => { // The primary key is the authority, and a writer that loses a position is - // retried at one that is still free. Nothing here may leave a hole: density - // is what lets a reader prove its copy of a log is complete. + // retried at one that is still free rather than abandoning the one it lost, + // so a burst of concurrent writers still numbers itself densely. for (const writers of [2, 8, 50]) { test(`keeps ${writers} concurrent writers dense`, async () => { const runId = await newSlotRun(); @@ -274,7 +274,7 @@ describe('Slot identity (Postgres integration)', () => { }, 60_000); } - test('proves completeness: the highest slot is the event count', async () => { + test('numbers a burst so the highest slot is the event count', async () => { const runId = await newSlotRun(); await Promise.all( Array.from({ length: 5 }, (_, index) => diff --git a/packages/world-vercel/src/events-v4.test.ts b/packages/world-vercel/src/events-v4.test.ts index 2139d8c3eb..5c3bda47dd 100644 --- a/packages/world-vercel/src/events-v4.test.ts +++ b/packages/world-vercel/src/events-v4.test.ts @@ -726,7 +726,7 @@ describe('createWorkflowRunEventV4 over HTTP', () => { // A slot-numbered run names its own event ids, so the id has to reach the // wire: the backend reads it from the frame meta and inserts it // conditionally. Dropped, the backend mints a ULID instead and the run - // silently loses the density its completeness check depends on. + // silently reverts to server-assigned identity mid-log. const origin = WORKFLOW_SERVER_URL_OVERRIDE || 'https://vercel-workflow.com'; const agent = new MockAgent(); diff --git a/packages/world/src/slot-identity.ts b/packages/world/src/slot-identity.ts index d761f91452..203005c719 100644 --- a/packages/world/src/slot-identity.ts +++ b/packages/world/src/slot-identity.ts @@ -7,11 +7,17 @@ * cursor that accepted a ULID keeps accepting a slot — and because the width is * fixed, lexicographic order is numeric order. * - * Slots are dense and start at 1. Density is the whole point of the scheme: it - * makes `events.length === maxSlot` a proof that a loaded log has no holes, - * which is something a server-minted ULID log can never offer. Zero is left - * unused because the inclusive lower fence for range queries over a run's - * events is the all-zero id. + * Slots start at 1 and are allocated contiguously, which is what makes + * contention explicit: two writers proposing one position cannot both win, and + * the loser is told which events it was missing. Zero is left unused because + * the inclusive lower fence for range queries over a run's events is the + * all-zero id. + * + * Allocation being contiguous does not make a published log gap-free, so + * `events.length === maxSlot` is not a completeness proof. A slot claimed by an + * operation that then fails for a reason of its own is never filled, and if a + * later slot has already been published the gap is permanent. Nothing may treat + * a missing slot as an event still on its way. * * A slot body decodes as a ULID *timestamp* of epoch 0 without erroring, so * nothing may read a time out of one. Use the event's own `createdAt` / From 435e52583639b85b4c7769d0cc35b8e12fcc28c8 Mon Sep 17 00:00:00 2001 From: Peter Wielander Date: Fri, 31 Jul 2026 06:01:09 -0700 Subject: [PATCH 31/35] fix(core): render structured error messages and name what a divergence waited for MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Three diagnostic gaps that together made replay divergence unreadable: - `composeLogLine` dropped `errorMessage` whenever the message did not already contain it, so a warn carrying an error alongside its own summary line logged the symptom and none of the diagnosis. - An unconsumable event named only itself. It is almost always an event whose entity this replay never issued, so the pending invocation queue is what distinguishes "never issued" from "issued under another id". - An inline step batch abandoned on a fenced claim logged neither which member was fenced nor how the others settled. The fence is per-write, so a batch can split: the rejected claim writes nothing while a sibling on a different slot commits. Also read a failed run's error through `returnValue()` in the race-repro harness — `runs.get` returns the raw serialized payload, so every corruption in the report carried a code and no message. Co-Authored-By: Claude Opus 5 --- .changeset/tidy-moons-observe.md | 5 +++ .../core/e2e/event-log-race-repro.test.ts | 35 +++++++++++++++++-- packages/core/src/log-format.test.ts | 26 +++++++++++++- packages/core/src/log-format.ts | 20 ++++++++--- packages/core/src/runtime.ts | 26 ++++++++++++-- packages/core/src/workflow.ts | 8 ++++- 6 files changed, 110 insertions(+), 10 deletions(-) create mode 100644 .changeset/tidy-moons-observe.md diff --git a/.changeset/tidy-moons-observe.md b/.changeset/tidy-moons-observe.md new file mode 100644 index 0000000000..9c6c5b7692 --- /dev/null +++ b/.changeset/tidy-moons-observe.md @@ -0,0 +1,5 @@ +--- +'@workflow/core': patch +--- + +Name the divergent event's pending invocations and the fenced member of an inline step batch in replay-divergence logs diff --git a/packages/core/e2e/event-log-race-repro.test.ts b/packages/core/e2e/event-log-race-repro.test.ts index e98f6fd10a..b40fed2ec3 100644 --- a/packages/core/e2e/event-log-race-repro.test.ts +++ b/packages/core/e2e/event-log-race-repro.test.ts @@ -373,6 +373,30 @@ function validateStormReturn(value: unknown): { return { stragglers }; } +/** + * Reads a terminal-failed run's error through `returnValue()`, which hydrates + * the stored payload into an Error. Returns undefined when the read itself + * fails — the outcome is already known from `errorCode`, so a missing message + * degrades the report rather than the classification. + */ +async function readFailureMessage( + run: Run +): Promise<{ name?: string; message?: string } | undefined> { + try { + await run.returnValue(); + return undefined; + } catch (err) { + if (WorkflowRunFailedError.is(err)) { + const cause = err.cause; + return { + name: cause instanceof Error ? cause.name : err.name, + message: cause instanceof Error ? cause.message : err.message, + }; + } + return undefined; + } +} + async function pollTerminalRun( run: Run, startedAt: number, @@ -412,13 +436,20 @@ async function pollTerminalRun( errorCode?: string; error?: { name?: string; message?: string }; }; + // `runs.get` hands back the raw serialized error payload, not an Error, so + // reading `.message` off it yields undefined and the report records the + // code with no diagnosis. Read the failure through the public + // return-value path, which hydrates it. For a corruption that message + // carries the divergent event and what the replay was waiting for, which + // is the whole reason to keep the report. + const hydrated = await readFailureMessage(run); return { ...base, outcome: classifyFailure(failure.errorCode), status: runData.status, errorCode: failure.errorCode, - errorMessage: failure.error?.message, - errorName: failure.error?.name, + errorMessage: hydrated?.message ?? failure.error?.message, + errorName: hydrated?.name ?? failure.error?.name, durationMs: Date.now() - startedAt, }; } diff --git a/packages/core/src/log-format.test.ts b/packages/core/src/log-format.test.ts index c673893f06..3b78208744 100644 --- a/packages/core/src/log-format.test.ts +++ b/packages/core/src/log-format.test.ts @@ -113,6 +113,29 @@ describe('composeLogLine', () => { `); }); + test('renders errorMessage when the message does not already carry it', () => { + // The replay-divergence warn writes its own summary line and passes the + // error only as metadata, so this is the sole place the divergent event's + // identity appears. Dropping it leaves the log naming a symptom with no + // way to tell which event diverged. + const out = composeLogLine( + PREFIX, + 'Workflow replay diverged; queueing a recovery replay before declaring the event log corrupted', + { + errorCode: 'REPLAY_DIVERGENCE', + errorMessage: + 'Replay could not consume event: eventType=step_created, correlationId=step_00000000000000000000000025.', + divergenceCount: 1, + } + ); + expect(out).toMatchInlineSnapshot(` + "[workflow-sdk] Workflow replay diverged; queueing a recovery replay before declaring the event log corrupted + code REPLAY_DIVERGENCE + error Replay could not consume event: eventType=step_created, correlationId=step_00000000000000000000000025. + divergenceCount 1" + `); + }); + test('falls back gracefully on machine names it cannot parse', () => { const out = composeLogLine(PREFIX, 'msg', { workflowRunId: 'wrun_X', @@ -156,7 +179,8 @@ describe('composeLogLine', () => { user error · Error run wrun_01ABC · myWorkflow (./workflows/x) step step_01XYZ · add (./workflows/x) - retry 4 attempts · 3 max retries" + retry 4 attempts · 3 max retries + error Transient failure" `); }); }); diff --git a/packages/core/src/log-format.ts b/packages/core/src/log-format.ts index d63e847e3f..b3daa3263c 100644 --- a/packages/core/src/log-format.ts +++ b/packages/core/src/log-format.ts @@ -36,7 +36,7 @@ export function composeLogLine( ): string { const [framing, ...rest] = message.split('\n'); const body = rest.join('\n'); - const fields = renderStructuredFields(framing ?? '', metadata); + const fields = renderStructuredFields(message, metadata); const trimmedBody = trimStackBody(body); const lines: string[] = [`${prefix} ${framing ?? ''}`]; @@ -46,19 +46,21 @@ export function composeLogLine( } function renderStructuredFields( - framing: string, + message: string, metadata: Record | undefined ): string | null { if (!metadata || Object.keys(metadata).length === 0) return null; // Drop fields that the message already encodes. We render framings and // stacks into the message string itself in step executor / combined runtime, so - // repeating them here would be pure noise. + // repeating them here would be pure noise. The whole message counts, not just + // its first line: callers that pass `${framing}\n${stack}` put the error's + // text in the stack's leading `Name: message` line. const redundant = new Set(); redundant.add('errorStack'); if ( typeof metadata.errorMessage === 'string' && - framing.includes(metadata.errorMessage as string) + message.includes(metadata.errorMessage as string) ) { redundant.add('errorMessage'); } @@ -130,6 +132,16 @@ function renderStructuredFields( lines.push(` ${kvKey('code')} ${Ansi.dim(errorCode)}`); } + // The message only duplicates the framing when the framing was built from + // the error itself (step executor, terminal run failures), and that case is + // already marked redundant above. Everywhere else — a warn that carries an + // error alongside its own summary line — this is the only place the error's + // own text appears, so dropping it loses the diagnosis. + const errorMessage = pickString(metadata, 'errorMessage'); + if (errorMessage && !redundant.has('errorMessage')) { + lines.push(` ${kvKey('error')} ${errorMessage}`); + } + const hint = pickString(metadata, 'hint'); if (hint) { lines.push(` ${Ansi.hint(hint)}`); diff --git a/packages/core/src/runtime.ts b/packages/core/src/runtime.ts index 19581650d2..6534fbfe15 100644 --- a/packages/core/src/runtime.ts +++ b/packages/core/src/runtime.ts @@ -2498,10 +2498,32 @@ export function workflowEntrypoint( // the sibling executions to settle first so no owned // body is in flight when the ack path runs. if (requiresFreshReplay(stepErr)) { - await Promise.allSettled(stepExecutionPromises); + const settled = await Promise.allSettled( + stepExecutionPromises + ); runtimeLogger.warn( 'Inline step claim rejected as stale; re-invoking run for a fresh replay', - { workflowRunId: runId, loopIteration } + { + workflowRunId: runId, + loopIteration, + // Which members of the batch were fenced and + // which committed. A fence is per-write, so a + // batch can be split: the rejected claim wrote + // nothing, but a sibling holding a different + // slot may have committed its step. That + // asymmetry is the shape to look for when a + // later replay cannot consume a step event. + batchSteps: inlineExecutions + .map( + (s, i) => + `${s.correlationId}:${settled[i]?.status === 'rejected' ? 'rejected' : 'settled'}` + ) + .join(', '), + errorMessage: + stepErr instanceof Error + ? stepErr.message + : String(stepErr), + } ); // The finally below resumes the replay budget // before this return completes. diff --git a/packages/core/src/workflow.ts b/packages/core/src/workflow.ts index 03f8d1e203..76fd5e00dd 100644 --- a/packages/core/src/workflow.ts +++ b/packages/core/src/workflow.ts @@ -220,9 +220,15 @@ export async function runWorkflow( updateTimestamp(+event.createdAt); }, onUnconsumedEvent: (event) => { + // Name what the replay was waiting for instead. An unconsumable event + // is almost always one whose entity this replay never issued, or + // issued under a different correlation ID; the pending invocation + // queue is the only place that distinction is visible, and without it + // the log names a symptom with no way to reach the cause. + const pending = [...workflowContext.invocationsQueue.keys()]; workflowDiscontinuation.reject( new ReplayDivergenceError( - `Replay could not consume event: eventType=${event.eventType}, correlationId=${event.correlationId}, eventId=${event.eventId}.`, + `Replay could not consume event: eventType=${event.eventType}, correlationId=${event.correlationId}, eventId=${event.eventId}. Pending invocations: ${pending.length > 0 ? pending.join(', ') : '(none)'}.`, { eventId: event.eventId } ) ); From 5cfc07dcdf6551d1401008bbc3e8500944bdfbd9 Mon Sep 17 00:00:00 2001 From: Peter Wielander Date: Fri, 31 Jul 2026 06:34:59 -0700 Subject: [PATCH 32/35] fix(core): reclaim a lost inline step slot instead of splitting the batch MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit An inline step's step_started claim is fenced per-write under slot identity, so a 409 only proves another writer took that write's number — routinely true, since the backend allocates outside events from the same next-free pointer the client reserves from. Abandoning the whole batch on it left the loser's events landing seconds later, after a whole later phase, in an order no single replay could consume. stepClaimFence keeps a watermark-guarded run on its single shared fence (a 412 does mean the view is stale, and the batch is meant to fail as a unit) and gives a slot-numbered run an in-place reclaim: merge the delta, reserve past it, re-claim. The reservation pointer is now absolute and only moves forward, so a merge cannot hand the retrying writer a slot a sibling is still in flight on. --- .changeset/inline-claim-reclaim.md | 5 + packages/core/src/logger.test.ts | 4 +- packages/core/src/runtime.ts | 35 +++-- packages/core/src/runtime/helpers.test.ts | 116 +++++++++++++++-- packages/core/src/runtime/helpers.ts | 121 ++++++++++++++---- .../core/src/runtime/step-executor.test.ts | 38 +++++- packages/core/src/runtime/step-executor.ts | 109 +++++++++------- 7 files changed, 333 insertions(+), 95 deletions(-) create mode 100644 .changeset/inline-claim-reclaim.md diff --git a/.changeset/inline-claim-reclaim.md b/.changeset/inline-claim-reclaim.md new file mode 100644 index 0000000000..9d57541efc --- /dev/null +++ b/.changeset/inline-claim-reclaim.md @@ -0,0 +1,5 @@ +--- +'@workflow/core': patch +--- + +Keep a batch of inline steps together when one of its event writes loses a race, instead of discarding the batch diff --git a/packages/core/src/logger.test.ts b/packages/core/src/logger.test.ts index 5560c9e213..78066923ca 100644 --- a/packages/core/src/logger.test.ts +++ b/packages/core/src/logger.test.ts @@ -148,6 +148,7 @@ describe('logger', () => { user error · FatalError run wrun_123 step step_456 + error boom hint: Move the call to a step function.", ], ] @@ -178,7 +179,8 @@ describe('logger', () => { user error · Error run wrun_abc step step_xyz - retry 4 attempts · 3 max retries", + retry 4 attempts · 3 max retries + error Transient failure", ], ] `); diff --git a/packages/core/src/runtime.ts b/packages/core/src/runtime.ts index 6534fbfe15..7f04514754 100644 --- a/packages/core/src/runtime.ts +++ b/packages/core/src/runtime.ts @@ -51,6 +51,7 @@ import { countStepStartedEvents } from './runtime/count-step-started-events.js'; import { appendUniqueEvents, eventCreateFenceFor, + stepClaimFence, getQueueOverhead, getWorkflowQueueName, handleHealthCheckMessage, @@ -2187,10 +2188,14 @@ export function workflowEntrypoint( // (retried over the reloaded log, or exhausted into // a queue re-invocation), AND the lazy step_started // claim of its next inline step, which is fenced too - // (threaded below via - // `eventCreateFence`; on rejection the batch is - // abandoned and re-invoked for a fresh replay, so a - // stale view can never commit a step). Hooks created + // (threaded below via `stepClaimFence`; on rejection + // the batch is abandoned and re-invoked for a fresh + // replay, so a stale view can never commit a step). + // A slot-numbered run gets there differently — the + // claim merges the missed events and retries in + // place, so the same events are observed without + // discarding the batch. See stepClaimFence. + // Hooks created // by THIS suspension are inside the delta (their // `hook_created` lands before the step-terminal // write), so only their `hook_received` responses @@ -2337,8 +2342,8 @@ export function workflowEntrypoint( // could claim — and commit — a step scheduled off a view // that misses an out-of-band event. One log for the // whole batch so each claim draws its own event slot; - // `eventCreateFenceFor` yields undefined for a run - // fenced neither way, leaving those claims as they were. + // `stepClaimFence` leaves the claims of a run fenced + // neither way exactly as they were. // // The suspension's own log, not a second one over the // same snapshot: its reservations are what the hook and @@ -2359,7 +2364,8 @@ export function workflowEntrypoint( // positional, so it has to be assigned before these // executions start racing each other, and the order // it is assigned in has to be replay-stable. - const eventCreateFence = eventCreateFenceFor( + const claimFence = stepClaimFence( + runId, inlineClaimLog, workflowRun.specVersion, { @@ -2445,7 +2451,7 @@ export function workflowEntrypoint( // see suppressOptimisticStart above. suppressOptimisticStart, runReadyBarrier, - eventCreateFence, + claimFence, ...(stepIndex === 0 && s.lazyStepInput !== undefined && latencyTracking @@ -2487,12 +2493,15 @@ export function workflowEntrypoint( ); } catch (stepErr) { // An incomplete-view rejection of an inline - // step_started claim (412 stale watermark, or 409 - // taken slot): the loaded view this batch was + // step_started claim: the loaded view this batch was // scheduled from is behind an out-of-band event (e.g. - // a received hook), so the claim was fenced by the - // guard and no step events were written. Abandon the - // batch — any optimistic body result is discarded by + // a received hook), so the claim was fenced and no + // step events were written. Under the watermark that + // is every 412; under slot identity `stepClaimFence` + // first merges the missed events and re-claims in + // place, so a 409 only arrives here once those + // retries are exhausted. Abandon the batch — any + // optimistic body result is discarded by // executeStep's reconciliation — and re-invoke for a // fresh replay that observes the new event. Wait for // the sibling executions to settle first so no owned diff --git a/packages/core/src/runtime/helpers.test.ts b/packages/core/src/runtime/helpers.test.ts index ca1a3ff4b5..1462804bd9 100644 --- a/packages/core/src/runtime/helpers.test.ts +++ b/packages/core/src/runtime/helpers.test.ts @@ -33,6 +33,7 @@ import { requiresFreshReplay, reserveSlot, stateUpdatedAtForCreate, + stepClaimFence, toMutableEventLog, withEventCreateFence, withPreconditionRetry, @@ -529,7 +530,7 @@ describe('slot bookkeeping', () => { // element is not necessarily its newest event. const log = toMutableEventLog([slotEvent(3), slotEvent(1)], 'c0'); expect(log.maxSlot).toBe(3); - expect(log.reserved).toBe(0); + expect(log.nextSlot).toBe(4); }); it('reports maxSlot 0 for an empty or ULID-numbered log', () => { @@ -566,21 +567,33 @@ describe('slot bookkeeping', () => { expect(log.events).toHaveLength(3); }); - it('raises maxSlot and drops reservations when a newer delta is merged in', () => { + it('raises the reservation pointer past a newer delta', () => { const log = toMutableEventLog([slotEvent(1)], 'c0'); reserveSlot(log); reserveSlot(log); - expect(log.reserved).toBe(2); + expect(log.nextSlot).toBe(4); mergeLoadedEvents(log, [slotEvent(2), slotEvent(5)]); expect(log.maxSlot).toBe(5); - // The merged events are the authority on which slots are taken, so the - // outstanding reservations (slots 2 and 3) are void. - expect(log.reserved).toBe(0); expect(reserveSlot(log)).toBe(6); }); + it('never rewinds the reservation pointer onto an outstanding slot', () => { + // A writer that loses its slot merges the delta and reserves again while + // its siblings are still in flight on theirs. Rewinding to `maxSlot + 1` + // would hand it slot 4, which a sibling already holds. + const log = toMutableEventLog([slotEvent(1)], 'c0'); + expect(reserveSlot(log)).toBe(2); + expect(reserveSlot(log)).toBe(3); + expect(reserveSlot(log)).toBe(4); + + mergeLoadedEvents(log, [slotEvent(2)]); + + expect(log.maxSlot).toBe(2); + expect(reserveSlot(log)).toBe(5); + }); + it('deduplicates merged events by id', () => { const log = toMutableEventLog([slotEvent(1)], 'c0'); mergeLoadedEvents(log, [slotEvent(1), slotEvent(2)]); @@ -639,7 +652,7 @@ describe('slot bookkeeping', () => { eventCreateFenceFor(log, SPEC_VERSION_SLOT_IDENTITY - 1, { extraEvents: 1, }); - expect(log.reserved).toBe(0); + expect(log.nextSlot).toBe(1); }); it('proposes no event id for a ULID-numbered run', () => { @@ -647,7 +660,7 @@ describe('slot bookkeeping', () => { const log = toMutableEventLog([], null); const fence = eventCreateFenceFor(log, SPEC_VERSION_SLOT_IDENTITY - 1); expect(fence?.eventId).toBeUndefined(); - expect(log.reserved).toBe(0); + expect(log.nextSlot).toBe(1); }); }); @@ -859,6 +872,93 @@ describe('withEventCreateFence', () => { }); }); +describe('stepClaimFence', () => { + const slotEvent = (slot: number) => makeEvent(slotEventId(slot)); + + beforeEach(() => { + eventsListMock.mockReset(); + }); + + it('numbers a batch in the order its claims were built, not the order they fire', async () => { + // The batch is built during replay and only starts racing afterwards, so + // the slot of each member has to be fixed at build time for the numbering + // to be replay-stable. + const log = toMutableEventLog([slotEvent(1)], 'c0'); + const first = stepClaimFence('wrun_test', log, SPEC_VERSION_SLOT_IDENTITY, { + extraEvents: 1, + }); + const second = stepClaimFence('wrun_test', log, SPEC_VERSION_SLOT_IDENTITY); + + const claimed: (string | undefined)[] = []; + const record = (fence?: { eventId?: string }) => { + claimed.push(fence?.eventId); + return Promise.resolve('ok'); + }; + // Fired in reverse: the numbering must not depend on it. + await second(record); + await first(record); + + expect(claimed).toEqual([slotEventId(4), slotEventId(3)]); + }); + + it('reclaims a lost slot in place, keeping the batch adjacent', async () => { + // The server allocates an outside event from the same next-free pointer + // the client reserves from, so losing a claim is routine. Abandoning the + // batch on it would leave this step's events far later in the log than its + // siblings' — an order no single replay can consume — and leave the lost + // slot permanently empty. + const log = toMutableEventLog([slotEvent(1)], 'c0'); + const claim = stepClaimFence('wrun_test', log, SPEC_VERSION_SLOT_IDENTITY); + const sibling = stepClaimFence( + 'wrun_test', + log, + SPEC_VERSION_SLOT_IDENTITY + ); + + const claimed: string[] = []; + const op = vi.fn(async (fence?: { eventId?: string }) => { + claimed.push(fence?.eventId as string); + if (claimed.length === 1) { + // An out-of-band hook took slot 2 while the batch was being built. + throw new SlotConflictError('taken', { + eventId: fence?.eventId as string, + events: [slotEvent(2)], + cursor: 'c1', + }); + } + return 'done'; + }); + + await expect(claim(op)).resolves.toBe('done'); + await expect(sibling(async (f) => f?.eventId)).resolves.toBe( + slotEventId(3) + ); + // Reclaimed above the merged event rather than propagating the conflict. + expect(claimed).toEqual([slotEventId(2), slotEventId(4)]); + }); + + it('leaves a ULID-numbered batch on one shared watermark, unretried', async () => { + // A 412 compares time, so every member of the batch carries the same fence + // value and the batch fails as a unit — which is what the caller's + // fresh-replay path expects. + const time = 1_700_000_000_000; + const log = toMutableEventLog([makeUlidEvent(time)], 'c0'); + const claim = stepClaimFence( + 'wrun_test', + log, + SPEC_VERSION_SLOT_IDENTITY - 1 + ); + const op = vi.fn(async () => { + throw new PreconditionFailedError('stale'); + }); + + await expect(claim(op)).rejects.toBeInstanceOf(PreconditionFailedError); + expect(op).toHaveBeenCalledTimes(1); + expect(op).toHaveBeenCalledWith({ stateUpdatedAt: time }); + expect(eventsListMock).not.toHaveBeenCalled(); + }); +}); + describe('requiresFreshReplay', () => { it('covers both fences, so neither numbering fails the run', () => { // Each fence reports an incomplete view in its own dialect. A caller that diff --git a/packages/core/src/runtime/helpers.ts b/packages/core/src/runtime/helpers.ts index 407f202635..c1a9248f32 100644 --- a/packages/core/src/runtime/helpers.ts +++ b/packages/core/src/runtime/helpers.ts @@ -662,11 +662,11 @@ export interface MutableEventLog { */ maxSlot: number; /** - * Slots handed out by `reserveSlot` past `maxSlot` whose events have not been - * merged back yet. Reset whenever the log is merged into, because the merged - * events are the authority on which slots are taken. + * Next slot `reserveSlot` will hand out. Absolute, and it only ever moves + * forward: a merge can raise it past the events it brought in, but must never + * lower it onto a slot already handed to a writer that is still in flight. */ - reserved: number; + nextSlot: number; } /** @@ -683,17 +683,18 @@ export function toMutableEventLog( cursor: string | null, slotFloor = 0 ): MutableEventLog { + const maxSlot = Math.max(maxSlotOf(events), slotFloor); return { events, cursor, - maxSlot: Math.max(maxSlotOf(events), slotFloor), - reserved: 0, + maxSlot, + nextSlot: maxSlot + 1, }; } /** * Merges loaded events into `log` in place, keeping `maxSlot` current and - * dropping outstanding reservations (the merged events supersede them). + * advancing the reservation pointer past the events merged in. */ export function mergeLoadedEvents( log: MutableEventLog, @@ -701,7 +702,7 @@ export function mergeLoadedEvents( ): void { appendUniqueEvents(log.events, events); log.maxSlot = Math.max(log.maxSlot, maxSlotOf(events)); - log.reserved = 0; + log.nextSlot = Math.max(log.nextSlot, log.maxSlot + 1); } /** @@ -713,10 +714,16 @@ export function mergeLoadedEvents( * in the flush would propose the same slot and all but one would conflict, on * every single flush. Operations are built in deterministic replay order, so * the slot each one draws is replay-stable too. + * + * The pointer is never rewound by a merge, only pushed forward. A writer that + * loses its slot merges the delta and reserves again while its siblings still + * hold theirs; rewinding to `maxSlot + 1` would hand it a sibling's slot and + * turn one conflict into a chain of them. */ export function reserveSlot(log: MutableEventLog): number { - log.reserved += 1; - return log.maxSlot + log.reserved; + const slot = log.nextSlot; + log.nextSlot = slot + 1; + return slot; } /** @@ -859,7 +866,7 @@ export interface EventCreateFence { /** * The fence for a create that is deliberately **not** retried in place, because * a rejection means the committed decision itself is stale and only a fresh - * replay can revise it (`run_completed`, an inline `step_started` claim). + * replay can revise it (`run_completed`). * * Claims a slot off `log` for a slot-numbered run — which counts as a * reservation, so a caller that fences several creates from one log gets a @@ -885,19 +892,80 @@ export function eventCreateFenceFor( options?: { extraEvents?: number } ): EventCreateFence | undefined { if (usesSlotIdentity(specVersion)) { - const maxSlot = log.maxSlot; - // The extra events sit below the one being created, matching the order a - // reader expects (a step is created before it starts) — so their slots are - // reserved first and the claim names the last of the run. - for (let i = 0; i < (options?.extraEvents ?? 0); i++) { - reserveSlot(log); - } - return { eventId: slotEventId(reserveSlot(log)), maxSlot }; + return reserveSlotFence(log, options?.extraEvents ?? 0); } const stateUpdatedAt = stateUpdatedAtForCreate(log.events, specVersion); return stateUpdatedAt !== undefined ? { stateUpdatedAt } : undefined; } +/** + * Reserves this write's slots off `log` and names the one the event itself + * takes. + * + * `extraEvents` sit below the one being created, matching the order a reader + * expects (a step is created before it starts), so their slots are reserved + * first and the claim names the last of the run — a World that writes a pair + * derives the lower id from the one it was given. + */ +function reserveSlotFence( + log: MutableEventLog, + extraEvents: number +): EventCreateFence { + const maxSlot = log.maxSlot; + for (let i = 0; i < extraEvents; i++) { + reserveSlot(log); + } + return { eventId: slotEventId(reserveSlot(log)), maxSlot }; +} + +/** + * Runs one event create under whichever fence its run uses, handling a lost + * claim however that run's scheme requires. + */ +export type FencedCreate = ( + op: (fence: EventCreateFence | undefined) => Promise +) => Promise; + +/** + * The fence for an inline step's `step_started` claim. + * + * A slot-numbered run retries a lost claim in place; a watermark-guarded run + * does not, and lets the rejection abandon the batch for a fresh replay. The + * asymmetry is in what a rejection proves, and it is the difference between a + * batch that stays contiguous and one that splits: + * + * - A 412 compares the *time* of the newest outside event, so every claim in a + * batch carries the same fence value and a stale view fails all of them. The + * batch is abandoned as a unit, nothing is written, and the fresh replay + * reschedules from a complete view. + * - A 409 only proves another writer took this write's *number*. That happens + * routinely without any staleness: the server allocates outside events from + * the same next-free pointer the client reserves from, so any outside event + * landing mid-batch takes the slot the batch's next claim is holding. Fencing + * the batch on it splits it — the loser writes nothing while its siblings + * commit, and the loser's events land far later in the log (or never), leaving + * an order no single replay can consume. Taking another number instead keeps + * the batch's events adjacent and its slots dense. + */ +export function stepClaimFence( + runId: string, + log: MutableEventLog, + specVersion: number | undefined, + options?: { extraEvents?: number } +): FencedCreate { + if (usesSlotIdentity(specVersion)) { + // Reserved here, synchronously, rather than when the claim fires: a batch's + // claims have to be numbered in replay order, and they only start racing + // each other afterwards. Retries re-reserve at that point by necessity — + // the merged delta has moved the log — but by then this write is the only + // one of the batch still choosing a slot. + const initialFence = reserveSlotFence(log, options?.extraEvents ?? 0); + return (op) => withSlotRetry(runId, log, op, { ...options, initialFence }); + } + const fence = eventCreateFenceFor(log, specVersion, options); + return (op) => op(fence); +} + /** * Runs a replay-context event creation that claims its own event slot. * @@ -917,15 +985,20 @@ export function eventCreateFenceFor( export async function withSlotRetry( runId: string, log: MutableEventLog, - op: (fence: EventCreateFence) => Promise + op: (fence: EventCreateFence) => Promise, + options?: { extraEvents?: number; initialFence?: EventCreateFence } ): Promise { for (let attempt = 0; ; attempt++) { // Claimed per attempt, not once up front: a merged delta moves the log's - // high-water mark, so the previous claim is stale by definition. - const maxSlot = log.maxSlot; - const eventId = slotEventId(reserveSlot(log)); + // high-water mark, so the previous claim is stale by definition. The first + // attempt can carry a slot the caller reserved earlier, for a caller whose + // numbering has to be assigned in a particular order (see stepClaimFence). + const fence = + (attempt === 0 ? options?.initialFence : undefined) ?? + reserveSlotFence(log, options?.extraEvents ?? 0); + const eventId = fence.eventId; try { - return await op({ eventId, maxSlot }); + return await op(fence); } catch (error) { if ( !SlotConflictError.is(error) || diff --git a/packages/core/src/runtime/step-executor.test.ts b/packages/core/src/runtime/step-executor.test.ts index 4cb6f6b8f1..8b4d99e746 100644 --- a/packages/core/src/runtime/step-executor.test.ts +++ b/packages/core/src/runtime/step-executor.test.ts @@ -4,7 +4,7 @@ import { join } from 'node:path'; import type { Event, World } from '@workflow/world'; import { SPEC_VERSION_CURRENT } from '@workflow/world'; import { createWorld } from '@workflow/world-local'; -import { afterEach, describe, expect, it } from 'vitest'; +import { afterEach, describe, expect, it, vi } from 'vitest'; import { registerStepFunction } from '../private.js'; import { dehydrateStepArguments } from '../serialization.js'; import { executeStep } from './step-executor.js'; @@ -164,3 +164,39 @@ describe('executeStep — retry ceiling (authoritativeAttempt)', () => { expect(ceilingFailures).toHaveLength(0); }); }); + +describe('executeStep — claim fence plumbing', () => { + afterEach(() => { + counter += 1; + }); + + it("runs the step_started claim under the caller's fence", async () => { + const world = makeWorld(); + const stepName = uniqueStepName(); + const { runId, stepId } = await setupRunningStep({ + world, + stepName, + onBody: () => {}, + }); + + // The fence rides in CreateEventParams, which world-local does not + // persist — so observe the call itself rather than the stored event. + const createSpy = vi.spyOn(world.events, 'create'); + + await executeStep({ + world, + workflowRunId: runId, + workflowName: 'wf', + workflowStartedAt: Date.now(), + stepId, + stepName, + claimFence: (op) => op({ stateUpdatedAt: 1_700_000_000_000 }), + }); + + const started = createSpy.mock.calls.filter( + ([, data]) => data.eventType === 'step_started' + ); + expect(started).toHaveLength(1); + expect(started[0]?.[2]?.stateUpdatedAt).toBe(1_700_000_000_000); + }); +}); diff --git a/packages/core/src/runtime/step-executor.ts b/packages/core/src/runtime/step-executor.ts index e1823cfb24..89a46947ff 100644 --- a/packages/core/src/runtime/step-executor.ts +++ b/packages/core/src/runtime/step-executor.ts @@ -43,7 +43,7 @@ import { isOptimisticInlineStartExplicitlyDisabled, } from './constants.js'; import { getPortLazy } from './get-port-lazy.js'; -import { type EventCreateFence, memoizeEncryptionKey } from './helpers.js'; +import { type FencedCreate, memoizeEncryptionKey } from './helpers.js'; import { computeStepLatencyEventData, type StepLatencyEventData, @@ -129,23 +129,27 @@ export interface StepExecutorParams { */ inlineDeltaSinceCursor?: string; /** - * Concurrency fence to attach to this step's `step_started` claim: the event - * slot the claim occupies, or the caller's replay snapshot (`stateUpdatedAt`, - * epoch ms of the latest event it loaded) for a run on the older numbering. + * Runs this step's `step_started` claim under its run's concurrency fence: + * the event slot the claim occupies, or the caller's replay snapshot + * (`stateUpdatedAt`, epoch ms of the latest event it loaded) for a run on the + * older numbering. * * On the lazy inline path the claim is the step's FIRST durable write (its * `step_created` is deferred), so without a fence it would be unguarded * entirely: a replay working from a stale view could claim — and then commit — * a step scheduled without observing an out-of-band event. A fencing World * rejects such a claim with `SlotConflictError` (409) or - * `PreconditionFailedError` (412); executeStep does NOT translate either - * rejection (re-claiming in place would still commit the stale schedule), so - * it propagates for the caller to abandon the batch and force a fresh replay. + * `PreconditionFailedError` (412). + * + * Whether a rejection is retried in place is the caller's decision, made per + * scheme — see `stepClaimFence`. Either way executeStep does NOT translate a + * rejection that reaches it, so an unretried one propagates for the caller to + * abandon the batch and force a fresh replay. * * Undefined when the caller has no snapshot, or when the watermark guard is * disabled on a run that uses it; Worlds that fence neither way ignore it. */ - eventCreateFence?: EventCreateFence; + claimFence?: FencedCreate; /** * Suppress optimistic inline start for this step regardless of * `WORKFLOW_OPTIMISTIC_INLINE_START` / `forceOptimisticStart`: take the @@ -253,6 +257,10 @@ export async function executeStep( stepName, } = params; const isVercel = process.env.VERCEL_URL !== undefined; + // Unfenced when the caller passes no fence — every World that fences + // ignores the field it does not understand, so this is the same create it + // was before either mechanism existed. + const runClaim: FencedCreate = params.claimFence ?? ((op) => op(undefined)); // Gate payload compression on the run's specVersion. const compression = (params.runSpecVersion ?? 0) >= SPEC_VERSION_SUPPORTS_COMPRESSION; @@ -543,27 +551,29 @@ export async function executeStep( // RSFS measures the run_started-to-POST stretch, and the barrier // wait IS part of that stretch under turbo. stepStartPostSentAtMs = Date.now(); - return world.events.create( - workflowRunId, - { - eventType: 'step_started', - specVersion: SPEC_VERSION_CURRENT, - correlationId: stepId, - eventData: { - stepName, - workflowName, - input: params.lazyStepInput, - // Inline-ownership stamp — see StepExecutorParams.ownerMessageId. - ...(params.ownerMessageId !== undefined - ? { ownerMessageId: params.ownerMessageId } - : {}), + // Fence the claim — see StepExecutorParams.claimFence. A rejection + // the fence does not retry surfaces via reconcileOptimisticStart as a + // non-translatable error: the body result is discarded and the + // rejection propagates to the caller. + return runClaim((fence) => + world.events.create( + workflowRunId, + { + eventType: 'step_started', + specVersion: SPEC_VERSION_CURRENT, + correlationId: stepId, + eventData: { + stepName, + workflowName, + input: params.lazyStepInput, + // Inline-ownership stamp — see StepExecutorParams.ownerMessageId. + ...(params.ownerMessageId !== undefined + ? { ownerMessageId: params.ownerMessageId } + : {}), + }, }, - }, - // Fence the claim — see StepExecutorParams.eventCreateFence. A 409/412 - // rejection surfaces via reconcileOptimisticStart as a - // non-translatable error: the body result is discarded and the - // rejection propagates to the caller. - params.eventCreateFence + fence + ) ); } ); @@ -604,26 +614,29 @@ export async function executeStep( ? { ownerMessageId: params.ownerMessageId } : {}; stepStartPostSentAtMs = Date.now(); - const startResult = await world.events.create( - workflowRunId, - { - eventType: 'step_started', - specVersion: SPEC_VERSION_CURRENT, - correlationId: stepId, - eventData: - params.lazyStepInput !== undefined - ? { - stepName, - workflowName, - input: params.lazyStepInput, - ...ownershipStamp, - } - : { stepName, ...ownershipStamp }, - }, - // Fence the claim — see StepExecutorParams.eventCreateFence. A 409/412 - // rejection is intentionally NOT translated by startErrorToResult - // below, so it propagates to the caller for a fresh replay. - params.eventCreateFence + // Fence the claim — see StepExecutorParams.claimFence. A rejection the + // fence does not retry is intentionally NOT translated by + // startErrorToResult below, so it propagates to the caller for a fresh + // replay. + const startResult = await runClaim((fence) => + world.events.create( + workflowRunId, + { + eventType: 'step_started', + specVersion: SPEC_VERSION_CURRENT, + correlationId: stepId, + eventData: + params.lazyStepInput !== undefined + ? { + stepName, + workflowName, + input: params.lazyStepInput, + ...ownershipStamp, + } + : { stepName, ...ownershipStamp }, + }, + fence + ) ); if (!startResult.step) { From f66196cdedd44156b0f4a08ae69726e4ffa005ca Mon Sep 17 00:00:00 2001 From: Peter Wielander Date: Fri, 31 Jul 2026 13:16:22 -0700 Subject: [PATCH 33/35] chore: untrack local e2e diagnostics dump --- .gitignore | 3 + e2e-diagnostics-nextjs-turbopack-local.json | 848 -------------------- 2 files changed, 3 insertions(+), 848 deletions(-) delete mode 100644 e2e-diagnostics-nextjs-turbopack-local.json diff --git a/.gitignore b/.gitignore index 8b7641f7d7..28b5575028 100644 --- a/.gitignore +++ b/.gitignore @@ -42,3 +42,6 @@ packages/swc-plugin-workflow/build-hash.json workbench/nextjs-*/public/.well-known/workflow workbench/sveltekit/static/.well-known/workflow + +# Local e2e diagnostics dumps +e2e-diagnostics-*.json diff --git a/e2e-diagnostics-nextjs-turbopack-local.json b/e2e-diagnostics-nextjs-turbopack-local.json deleted file mode 100644 index 73a3cda9df..0000000000 --- a/e2e-diagnostics-nextjs-turbopack-local.json +++ /dev/null @@ -1,848 +0,0 @@ -[ - { - "testName": "addTenWorkflow", - "runId": "wrun_01KYWTGT53YMY8BX4JKWHYVPD9", - "timestamp": "2026-07-31T19:30:49.649Z", - "dashboardUrl": null - }, - { - "testName": "addTenWorkflow", - "runId": "wrun_01KYWTGWX1Z0ZV3HYPFH7SKYCC", - "timestamp": "2026-07-31T19:30:52.452Z", - "dashboardUrl": null - }, - { - "testName": "deploymentId: 'latest' is a no-op in non-Vercel worlds", - "runId": "wrun_01KYWTGZE8309234ZWEJNK29QB", - "timestamp": "2026-07-31T19:30:55.051Z", - "dashboardUrl": null - }, - { - "testName": "wellKnownAgentWorkflow (.well-known/agent)", - "runId": "wrun_01KYWTH207G4MECBEGA07TJ08W", - "timestamp": "2026-07-31T19:30:57.674Z", - "dashboardUrl": null - }, - { - "testName": "should work with react rendering in step", - "runId": "wrun_01KYWTH2ZQCE82XFR5GER6HMNA", - "timestamp": "2026-07-31T19:30:58.687Z", - "dashboardUrl": null - }, - { - "testName": "promiseAllWorkflow", - "runId": "wrun_01KYWTH3ZATA1AMTRNKR26AEEJ", - "timestamp": "2026-07-31T19:30:59.693Z", - "dashboardUrl": null - }, - { - "testName": "promiseRaceWorkflow", - "runId": "wrun_01KYWTH6XDT15ZVPWGYW1W1J46", - "timestamp": "2026-07-31T19:31:02.708Z", - "dashboardUrl": null - }, - { - "testName": "promiseAnyWorkflow", - "runId": "wrun_01KYWTHVHXDYNC7JTAF25QGT3X", - "timestamp": "2026-07-31T19:31:23.876Z", - "dashboardUrl": null - }, - { - "testName": "importedStepOnlyWorkflow", - "runId": "wrun_01KYWTJ2F4SDBP0WAEE3FQXTA3", - "timestamp": "2026-07-31T19:31:30.970Z", - "dashboardUrl": null - }, - { - "testName": "readableStreamWorkflow", - "runId": "wrun_01KYWTJ3GCRERJ80FMM6C9WK61", - "timestamp": "2026-07-31T19:31:32.027Z", - "dashboardUrl": null - }, - { - "testName": "hookWorkflow", - "runId": "wrun_01KYWTJCGAQTY2S8DHXNKMTGAH", - "timestamp": "2026-07-31T19:31:41.249Z", - "dashboardUrl": null - }, - { - "testName": "hookWorkflow is not resumable via public webhook endpoint", - "runId": "wrun_01KYWTJE0N3Q65FDEMJ60486FC", - "timestamp": "2026-07-31T19:31:42.767Z", - "dashboardUrl": null - }, - { - "testName": "webhookWorkflow", - "runId": "wrun_01KYWTJHWC6H21T1PBSHHG11R0", - "timestamp": "2026-07-31T19:31:46.745Z", - "dashboardUrl": null - }, - { - "testName": "parallelStepsThenWebhookWorkflow - no hook_conflict from same-tick replay race", - "runId": "wrun_01KYWTJNEDJWC6TY0NXR6XNH62", - "timestamp": "2026-07-31T19:31:50.398Z", - "dashboardUrl": null - }, - { - "testName": "sleepingWorkflow", - "runId": "wrun_01KYWTK9MFDYNBSAX5CQ61HCY0", - "timestamp": "2026-07-31T19:32:11.048Z", - "dashboardUrl": null - }, - { - "testName": "parallelSleepWorkflow", - "runId": "wrun_01KYWTKMEEB8W3007D7Q9APF9Z", - "timestamp": "2026-07-31T19:32:22.098Z", - "dashboardUrl": null - }, - { - "testName": "sleepWinsRaceWorkflow", - "runId": "wrun_01KYWTKPD8KK1TP8KRMX97HCTW", - "timestamp": "2026-07-31T19:32:24.108Z", - "dashboardUrl": null - }, - { - "testName": "stepWinsRaceWorkflow", - "runId": "wrun_01KYWTKRBZHA0PAHBYH9MTMHX4", - "timestamp": "2026-07-31T19:32:26.115Z", - "dashboardUrl": null - }, - { - "testName": "nullByteWorkflow", - "runId": "wrun_01KYWTKTASR8KS1S97MFKHH6AC", - "timestamp": "2026-07-31T19:32:28.130Z", - "dashboardUrl": null - }, - { - "testName": "workflowAndStepMetadataWorkflow", - "runId": "wrun_01KYWTKVADTSZD3QFQ1QDYFX01", - "timestamp": "2026-07-31T19:32:29.137Z", - "dashboardUrl": null - }, - { - "testName": "no startIndex (reads all chunks)", - "runId": "wrun_01KYWTKW9XYD8K1B7TK1PWFVRD", - "timestamp": "2026-07-31T19:32:30.147Z", - "dashboardUrl": null - }, - { - "testName": "positive startIndex (skips first chunk)", - "runId": "wrun_01KYWTM2WJ4E2VAKKBA3RFB6KS", - "timestamp": "2026-07-31T19:32:36.885Z", - "dashboardUrl": null - }, - { - "testName": "negative startIndex (reads from end)", - "runId": "wrun_01KYWTM8RQQFS4KYD92DXDQVJA", - "timestamp": "2026-07-31T19:32:42.909Z", - "dashboardUrl": null - }, - { - "testName": "getTailIndex returns correct index after stream completes", - "runId": "wrun_01KYWTMEMXEM02429V1HGVWJSW", - "timestamp": "2026-07-31T19:32:48.929Z", - "dashboardUrl": null - }, - { - "testName": "getTailIndex returns -1 before any chunks are written", - "runId": "wrun_01KYWTMMGZ0EXXTPC7Q0CP4NVK", - "timestamp": "2026-07-31T19:32:54.951Z", - "dashboardUrl": null - }, - { - "testName": "getChunks returns same content as reading the stream", - "runId": "wrun_01KYWTMMH9ZKJ3RMQ06H6GVKYF", - "timestamp": "2026-07-31T19:32:54.961Z", - "dashboardUrl": null - }, - { - "testName": "outputStreamInsideStepWorkflow - getWritable() called inside step functions", - "runId": "wrun_01KYWTMVCPH5ZQ127WFA7DR4DQ", - "timestamp": "2026-07-31T19:33:01.977Z", - "dashboardUrl": null - }, - { - "testName": "utf8StreamWorkflow", - "runId": "wrun_01KYWTN1018KV6MGDG65CF52G8", - "timestamp": "2026-07-31T19:33:07.716Z", - "dashboardUrl": null - }, - { - "testName": "writableForwardedFromWorkflowWorkflow", - "runId": "wrun_01KYWTN3BBG0RVQJWB7YCEW80B", - "timestamp": "2026-07-31T19:33:10.126Z", - "dashboardUrl": null - }, - { - "testName": "writableForwardedFromStepWorkflow", - "runId": "wrun_01KYWTN4H8D8Y40MNV6MK9JA72", - "timestamp": "2026-07-31T19:33:11.338Z", - "dashboardUrl": null - }, - { - "testName": "fetchWorkflow", - "runId": "wrun_01KYWTN5KWPKK4552YGNZ8D672", - "timestamp": "2026-07-31T19:33:12.447Z", - "dashboardUrl": null - }, - { - "testName": "promiseRaceStressTestWorkflow", - "runId": "wrun_01KYWTN6K9NCKYF6F4HZDTYJ6P", - "timestamp": "2026-07-31T19:33:13.454Z", - "dashboardUrl": null - }, - { - "testName": "nested function calls preserve message and stack trace", - "runId": "wrun_01KYWTNV5B5Q920TCE4HKQA1P5", - "timestamp": "2026-07-31T19:33:34.528Z", - "dashboardUrl": null - }, - { - "testName": "cross-file imports preserve message and stack trace", - "runId": "wrun_01KYWTP0HG449F33YGC8N3VZ15", - "timestamp": "2026-07-31T19:33:40.020Z", - "dashboardUrl": null - }, - { - "testName": "basic step error preserves message and stack trace", - "runId": "wrun_01KYWTP8DMRKPH4YVREB8883ZX", - "timestamp": "2026-07-31T19:33:48.093Z", - "dashboardUrl": null - }, - { - "testName": "cross-file step error preserves message and function names in stack", - "runId": "wrun_01KYWTPT1K314ZS28VXBAMJD49", - "timestamp": "2026-07-31T19:34:06.140Z", - "dashboardUrl": null - }, - { - "testName": "regular Error retries until success", - "runId": "wrun_01KYWTQ136BMTVANXKY45RB3FC", - "timestamp": "2026-07-31T19:34:13.361Z", - "dashboardUrl": null - }, - { - "testName": "FatalError fails immediately without retries", - "runId": "wrun_01KYWTQ7DQ6AHDBPAAQ08074P7", - "timestamp": "2026-07-31T19:34:19.835Z", - "dashboardUrl": null - }, - { - "testName": "RetryableError respects custom retryAfter delay", - "runId": "wrun_01KYWTQF8E1T2R49Z0PA7ECJ9M", - "timestamp": "2026-07-31T19:34:27.866Z", - "dashboardUrl": null - }, - { - "testName": "maxRetries=0 disables retries", - "runId": "wrun_01KYWTQT136FT5047CJEC29SH2", - "timestamp": "2026-07-31T19:34:38.887Z", - "dashboardUrl": null - }, - { - "testName": "FatalError can be caught and detected with FatalError.is()", - "runId": "wrun_01KYWTQV0H9R02DPQZ4NNTBYGF", - "timestamp": "2026-07-31T19:34:39.893Z", - "dashboardUrl": null - }, - { - "testName": "step throw round-trips FatalError with cause chain to workflow catch", - "runId": "wrun_01KYWTR0JNPEGQWP23K0860MND", - "timestamp": "2026-07-31T19:34:45.595Z", - "dashboardUrl": null - }, - { - "testName": "workflow throw round-trips FatalError + cause through run_failed event", - "runId": "wrun_01KYWTR57Z8SFEDC2F2E1AB53V", - "timestamp": "2026-07-31T19:34:50.371Z", - "dashboardUrl": null - }, - { - "testName": "workflow throw of a non-Error value round-trips verbatim as cause", - "runId": "wrun_01KYWTR95MW35V8DZZ79BQHEVX", - "timestamp": "2026-07-31T19:34:54.393Z", - "dashboardUrl": null - }, - { - "testName": "step throw of a non-Error value preserves it as cause on the wrapping FatalError", - "runId": "wrun_01KYWTRCYWR0B9RXDF3QZ676E6", - "timestamp": "2026-07-31T19:34:58.273Z", - "dashboardUrl": null - }, - { - "testName": "WorkflowNotRegisteredError fails the run when workflow does not exist", - "runId": "wrun_01KYWTRG1WPWABASTGR7PVFG8H", - "timestamp": "2026-07-31T19:35:01.442Z", - "dashboardUrl": null - }, - { - "testName": "StepNotRegisteredError fails the step but workflow can catch it", - "runId": "wrun_01KYWTRK81ZETDCHGTAMAERV0P", - "timestamp": "2026-07-31T19:35:04.710Z", - "dashboardUrl": null - }, - { - "testName": "StepNotRegisteredError fails the run when not caught in workflow", - "runId": "wrun_01KYWTRRGPRHS7AVTGTG71MA4D", - "timestamp": "2026-07-31T19:35:10.110Z", - "dashboardUrl": null - }, - { - "testName": "hookCleanupTestWorkflow - hook token reuse after workflow completion", - "runId": "wrun_01KYWTRSJJ14ZPKGZZ1NM10R2P", - "timestamp": "2026-07-31T19:35:11.190Z", - "dashboardUrl": null - }, - { - "testName": "hookCleanupTestWorkflow - hook token reuse after workflow completion", - "runId": "wrun_01KYWTRTT9KG8R6PSJPZTKDZ70", - "timestamp": "2026-07-31T19:35:12.461Z", - "dashboardUrl": null - }, - { - "testName": "concurrent hook token conflict - two workflows cannot use the same hook token simultaneously", - "runId": "wrun_01KYWTS12RRCJQMRWZVNFFRXB8", - "timestamp": "2026-07-31T19:35:18.877Z", - "dashboardUrl": null - }, - { - "testName": "concurrent hook token conflict - two workflows cannot use the same hook token simultaneously", - "runId": "wrun_01KYWTS1AT7M9Z6YN7K946BNT3", - "timestamp": "2026-07-31T19:35:19.133Z", - "dashboardUrl": null - }, - { - "testName": "hookGetConflictWorkflow - awaiting hook.getConflict() registers hook without payload", - "runId": "wrun_01KYWTS8ETNFDTFHGVBT3962PK", - "timestamp": "2026-07-31T19:35:26.430Z", - "dashboardUrl": null - }, - { - "testName": "'hookGetConflictWithPriorStepWorkflow' - hook.getConflict() does not block step execution", - "runId": "wrun_01KYWTS9E9N19K42B08N4QGTF6", - "timestamp": "2026-07-31T19:35:27.437Z", - "dashboardUrl": null - }, - { - "testName": "'hookGetConflictWithParallelStepWorkfl…' - hook.getConflict() does not block step execution", - "runId": "wrun_01KYWTSADVMTC1KA9JSJDGVP7N", - "timestamp": "2026-07-31T19:35:28.449Z", - "dashboardUrl": null - }, - { - "testName": "hookGetConflictThenStepParallelWorkflow - hook.getConflict() continuation step runs alongside other steps", - "runId": "wrun_01KYWTSBDEHP6T5E2QTV1HT105", - "timestamp": "2026-07-31T19:35:29.458Z", - "dashboardUrl": null - }, - { - "testName": "hookGetConflictWorkflow - hook.getConflict() resolves with the conflicting run when token is already registered", - "runId": "wrun_01KYWTSP5RG0TRGZHK55B6PZY5", - "timestamp": "2026-07-31T19:35:40.481Z", - "dashboardUrl": null - }, - { - "testName": "hookGetConflictWorkflow - hook.getConflict() resolves with the conflicting run when token is already registered", - "runId": "wrun_01KYWTSPDXXQ0MBYVY2BH2Y1VY", - "timestamp": "2026-07-31T19:35:40.739Z", - "dashboardUrl": null - }, - { - "testName": "hookClaimOnlyMutexWorkflow - hook works as a pure run mutex without payload data", - "runId": "wrun_01KYWTSTYCHBMGS8S7FTEW4V81", - "timestamp": "2026-07-31T19:35:45.367Z", - "dashboardUrl": null - }, - { - "testName": "hookClaimOnlyMutexWorkflow - hook works as a pure run mutex without payload data", - "runId": "wrun_01KYWTSV6M0G7DT6A05NY0HYFJ", - "timestamp": "2026-07-31T19:35:45.625Z", - "dashboardUrl": null - }, - { - "testName": "hookClaimOnlyMutexWorkflow - hook works as a pure run mutex without payload data", - "runId": "wrun_01KYWTT9W6X75NJTAQ8CYVR6G4", - "timestamp": "2026-07-31T19:36:00.649Z", - "dashboardUrl": null - }, - { - "testName": "hookAdoptOwnerResultWorkflow - duplicate adopts the owner result via conflict.returnValue", - "runId": "wrun_01KYWTTAVK7FBSX4D3MRQAMGKQ", - "timestamp": "2026-07-31T19:36:01.654Z", - "dashboardUrl": null - }, - { - "testName": "hookAdoptOwnerResultWorkflow - duplicate adopts the owner result via conflict.returnValue", - "runId": "wrun_01KYWTTB3H2DH8X898J6HT87YK", - "timestamp": "2026-07-31T19:36:01.908Z", - "dashboardUrl": null - }, - { - "testName": "hookSignalOwnerWorkflow - duplicate forwards its payload to the owner via resumeHook", - "runId": "wrun_01KYWTTCJZFQQVYTFC3DGQ9AZ6", - "timestamp": "2026-07-31T19:36:03.428Z", - "dashboardUrl": null - }, - { - "testName": "hookSignalOwnerWorkflow - duplicate forwards its payload to the owner via resumeHook", - "runId": "wrun_01KYWTTCTZASVMDC9MZ9ZGZKMT", - "timestamp": "2026-07-31T19:36:03.682Z", - "dashboardUrl": null - }, - { - "testName": "hookSupersedeOwnerWorkflow - duplicate cancels the owner and claims the released token", - "runId": "wrun_01KYWTTDTC0X9R0J8Y6NQN9GY6", - "timestamp": "2026-07-31T19:36:04.689Z", - "dashboardUrl": null - }, - { - "testName": "hookSupersedeOwnerWorkflow - duplicate cancels the owner and claims the released token", - "runId": "wrun_01KYWTTE2D2AZMN7S11B7NHQVD", - "timestamp": "2026-07-31T19:36:04.945Z", - "dashboardUrl": null - }, - { - "testName": "resume-or-start route pattern - resumeHook retried after start() reaches the new run", - "runId": "wrun_01KYWTTHVYA70TF6TMD5VA5TMY", - "timestamp": "2026-07-31T19:36:08.834Z", - "dashboardUrl": null - }, - { - "testName": "hookDisposeTestWorkflow - hook token reuse after explicit disposal while workflow still running", - "runId": "wrun_01KYWTTK3KSBK34YQG0AKVDB5S", - "timestamp": "2026-07-31T19:36:10.104Z", - "dashboardUrl": null - }, - { - "testName": "hookDisposeTestWorkflow - hook token reuse after explicit disposal while workflow still running", - "runId": "wrun_01KYWTTMBBY2H750S8BFZMW4NA", - "timestamp": "2026-07-31T19:36:11.376Z", - "dashboardUrl": null - }, - { - "testName": "hookTokenReuseLoopWorkflow - same run recreates a hook with the same token after dispose()", - "runId": "wrun_01KYWTTZ681MXD232MBCFC31QQ", - "timestamp": "2026-07-31T19:36:22.478Z", - "dashboardUrl": null - }, - { - "testName": "stepFunctionPassingWorkflow - step function references can be passed as arguments (without closure vars)", - "runId": "wrun_01KYWTV36WNVT61JF7RHTRMFD6", - "timestamp": "2026-07-31T19:36:26.592Z", - "dashboardUrl": null - }, - { - "testName": "stepFunctionWithClosureWorkflow - step function with closure variables passed as argument", - "runId": "wrun_01KYWTV81NV7Y86Q8BBNARSF4M", - "timestamp": "2026-07-31T19:36:31.545Z", - "dashboardUrl": null - }, - { - "testName": "closureVariableWorkflow - nested step functions with closure variables", - "runId": "wrun_01KYWTVASK303FKWMH710NR50N", - "timestamp": "2026-07-31T19:36:34.358Z", - "dashboardUrl": null - }, - { - "testName": "spawnWorkflowFromStepWorkflow - spawning a child workflow using start() inside a step", - "runId": "wrun_01KYWTVBS46NC2RW292EM5CQ4E", - "timestamp": "2026-07-31T19:36:35.379Z", - "dashboardUrl": null - }, - { - "testName": "runClassSerializationWorkflow - Run instances serialize across workflow/step boundaries", - "runId": "wrun_01KYWTVH8PVZGW79CR72G7QCAA", - "timestamp": "2026-07-31T19:36:40.988Z", - "dashboardUrl": null - }, - { - "testName": "startFromWorkflow - calling start() directly inside a workflow function with hook communication", - "runId": "wrun_01KYWTVP84HMHTD6WHA2YEYG82", - "timestamp": "2026-07-31T19:36:46.087Z", - "dashboardUrl": null - }, - { - "testName": "startFromWorkflow - calling start() directly inside a workflow function with hook communication", - "runId": "wrun_01KYWTVP84HMHTD6WHA2YEYG82", - "timestamp": "2026-07-31T19:36:46.087Z", - "dashboardUrl": null - }, - { - "testName": "startFromWorkflow - calling start() directly inside a workflow function with hook communication", - "runId": "wrun_01KYWTVP9FPHZH0VBC29PX675R", - "timestamp": "2026-07-31T19:36:47.090Z", - "dashboardUrl": null - }, - { - "testName": "fibonacciWorkflow - recursive workflow composition via start()", - "runId": "wrun_01KYWTVQ7KVHN8DNS4MS7EB3CS", - "timestamp": "2026-07-31T19:36:47.098Z", - "dashboardUrl": null - }, - { - "testName": "fibonacciWorkflow - recursive workflow composition via start()", - "runId": "wrun_01KYWTVQ7KVHN8DNS4MS7EB3CS", - "timestamp": "2026-07-31T19:36:47.098Z", - "dashboardUrl": null - }, - { - "testName": "pathsAliasWorkflow - TypeScript path aliases resolve correctly", - "runId": "wrun_01KYWTVZRP0MMM4ZY52VNJCK2X", - "timestamp": "2026-07-31T19:36:55.838Z", - "dashboardUrl": null - }, - { - "testName": "Calculator.calculate - static workflow method using static step methods from another class", - "runId": "wrun_01KYWTW2WD30KDSERSAEEMYQTZ", - "timestamp": "2026-07-31T19:36:59.025Z", - "dashboardUrl": null - }, - { - "testName": "AllInOneService.processNumber - static workflow method using sibling static step methods", - "runId": "wrun_01KYWTW6VKT6FVD6A9A5WZPBG7", - "timestamp": "2026-07-31T19:37:03.104Z", - "dashboardUrl": null - }, - { - "testName": "ChainableService.processWithThis - static step methods using `this` to reference the class", - "runId": "wrun_01KYWTWAG87PPD6TGE58KN3YYA", - "timestamp": "2026-07-31T19:37:06.833Z", - "dashboardUrl": null - }, - { - "testName": "thisSerializationWorkflow - step function invoked with .call() and .apply()", - "runId": "wrun_01KYWTWFGCDV5YRGRBC3CHHSVH", - "timestamp": "2026-07-31T19:37:11.955Z", - "dashboardUrl": null - }, - { - "testName": "customSerializationWorkflow - custom class serialization with WORKFLOW_SERIALIZE/WORKFLOW_DESERIALIZE", - "runId": "wrun_01KYWTWKHFD2BNB5PYSM0QTMC1", - "timestamp": "2026-07-31T19:37:16.083Z", - "dashboardUrl": null - }, - { - "testName": "instanceMethodStepWorkflow - instance methods with \"use step\" directive", - "runId": "wrun_01KYWTWQ4FNK1DV94S72HF5NSJ", - "timestamp": "2026-07-31T19:37:19.763Z", - "dashboardUrl": null - }, - { - "testName": "crossContextSerdeWorkflow - classes defined in step code are deserializable in workflow context", - "runId": "wrun_01KYWTWWSRA4RRQ9BN4XK33S0W", - "timestamp": "2026-07-31T19:37:25.566Z", - "dashboardUrl": null - }, - { - "testName": "errorSubclassRoundTripWorkflow - first-class Error subclasses survive every serialization boundary", - "runId": "wrun_01KYWTX02EAN0P7Z6XPZME7AQX", - "timestamp": "2026-07-31T19:37:28.919Z", - "dashboardUrl": null - }, - { - "testName": "stepFunctionAsStartArgWorkflow - step function reference passed as start() argument", - "runId": "wrun_01KYWTX1265YFEX0EDDAH4YSB9", - "timestamp": "2026-07-31T19:37:29.935Z", - "dashboardUrl": null - }, - { - "testName": "cancelRun - cancelling a running workflow", - "runId": "wrun_01KYWTX4M4ZH02XMM25GGVAE1B", - "timestamp": "2026-07-31T19:37:33.577Z", - "dashboardUrl": null - }, - { - "testName": "cancelRun via CLI - cancelling a running workflow", - "runId": "wrun_01KYWTX7DNCPQA2GPFQNGQNRJ0", - "timestamp": "2026-07-31T19:37:36.444Z", - "dashboardUrl": null - }, - { - "testName": "addTenWorkflow via pages router", - "runId": "wrun_01KYWTXD09P8XHTQGMXVWTPEED", - "timestamp": "2026-07-31T19:37:42.269Z", - "dashboardUrl": null - }, - { - "testName": "promiseAllWorkflow via pages router", - "runId": "wrun_01KYWTXE3GJ22QE6RRK1NQ1VPB", - "timestamp": "2026-07-31T19:37:43.330Z", - "dashboardUrl": null - }, - { - "testName": "sleepingWorkflow via pages router", - "runId": "wrun_01KYWTXH33GH4JBD33DK7WEMKA", - "timestamp": "2026-07-31T19:37:46.367Z", - "dashboardUrl": null - }, - { - "testName": "plainModuleDoneHook resumed via plain API route (o2flow shape)", - "runId": "wrun_01KYWTXVWA8C8CVXJ95V5N3QR4", - "timestamp": "2026-07-31T19:37:57.394Z", - "dashboardUrl": null - }, - { - "testName": "hookWithSleepWorkflow - hook payloads delivered correctly with concurrent sleep", - "runId": "wrun_01KYWTXX5N5M947PY128TVP0V6", - "timestamp": "2026-07-31T19:37:58.719Z", - "dashboardUrl": null - }, - { - "testName": "hookWithSleepFinalStepWorkflow - step only on final payload", - "runId": "wrun_01KYWTY1470C7T0W6DPY1WB2M8", - "timestamp": "2026-07-31T19:38:02.764Z", - "dashboardUrl": null - }, - { - "testName": "sleepInLoopWorkflow - sleep inside loop with steps actually delays each iteration", - "runId": "wrun_01KYWTY4MXFRQH3D08N6B33NZ4", - "timestamp": "2026-07-31T19:38:06.370Z", - "dashboardUrl": null - }, - { - "testName": "sleepWithSequentialStepsWorkflow - sequential steps work with concurrent sleep (control)", - "runId": "wrun_01KYWTYBG5HGHG076R85GN5M8G", - "timestamp": "2026-07-31T19:38:13.386Z", - "dashboardUrl": null - }, - { - "testName": "abortTimeoutWorkflow: timeout cancels long-running step", - "runId": "wrun_01KYWTYEMCSGRDE6ZP43T1AG5T", - "timestamp": "2026-07-31T19:38:16.592Z", - "dashboardUrl": null - }, - { - "testName": "abortParallelWorkflow: abort cancels all parallel steps", - "runId": "wrun_01KYWTYJHWZQF85FQ2P5KHK14J", - "timestamp": "2026-07-31T19:38:20.615Z", - "dashboardUrl": null - }, - { - "testName": "abortFromStepWorkflow: step abort cancels an in-flight sibling step", - "runId": "wrun_01KYWTYPFFY1YN6MG939K7WJTC", - "timestamp": "2026-07-31T19:38:24.627Z", - "dashboardUrl": null - }, - { - "testName": "abortAlreadyAbortedWorkflow: pre-aborted signal seen by step", - "runId": "wrun_01KYWTYRE8CVM07RBRP5JT3Z7R", - "timestamp": "2026-07-31T19:38:26.641Z", - "dashboardUrl": null - }, - { - "testName": "abortReasonWorkflow: abort reason preserved across boundaries", - "runId": "wrun_01KYWTYSDW8FJE1ZK195Y2PDKW", - "timestamp": "2026-07-31T19:38:27.654Z", - "dashboardUrl": null - }, - { - "testName": "abortAfterCompletionWorkflow: abort after step completes is a no-op", - "runId": "wrun_01KYWTYWC3STAPMJQ4QTFSMGTV", - "timestamp": "2026-07-31T19:38:30.663Z", - "dashboardUrl": null - }, - { - "testName": "abortViaHookWorkflow: external hook triggers abort on in-flight step", - "runId": "wrun_01KYWTYXBKMDANRQ5SSG0JSY2C", - "timestamp": "2026-07-31T19:38:31.677Z", - "dashboardUrl": null - }, - { - "testName": "abortExternalSignalWorkflow: signal passed as workflow input", - "runId": "wrun_01KYWTYYKTDFEQTWF98360VJJ8", - "timestamp": "2026-07-31T19:38:32.962Z", - "dashboardUrl": null - }, - { - "testName": "abortExternalSignalInFlightWorkflow: external abort fires mid-flight, propagates to nested steps", - "runId": "wrun_01KYWTYZKD92D0N50HTAJVAMC1", - "timestamp": "2026-07-31T19:38:33.968Z", - "dashboardUrl": null - }, - { - "testName": "abortAnyInWorkflowWorkflow: AbortSignal.any composes signals inside the workflow VM", - "runId": "wrun_01KYWTZ2HEFYASWMGQC6Z520RT", - "timestamp": "2026-07-31T19:38:36.979Z", - "dashboardUrl": null - }, - { - "testName": "abortAnyInStepWorkflow: AbortSignal.any inside a step composes deserialized signals", - "runId": "wrun_01KYWTZ3GZ00AADP380JWS0DBN", - "timestamp": "2026-07-31T19:38:37.987Z", - "dashboardUrl": null - }, - { - "testName": "abortSurvivesReplayWorkflow: controller state consistent across replay", - "runId": "wrun_01KYWTZ6F3TPX1Z9AT73MR6WMC", - "timestamp": "2026-07-31T19:38:41.007Z", - "dashboardUrl": null - }, - { - "testName": "abortThrowIfAbortedWorkflow: throwIfAborted causes FatalError, no retries", - "runId": "wrun_01KYWTZ8E3DPT451Z6HARJRBDR", - "timestamp": "2026-07-31T19:38:43.021Z", - "dashboardUrl": null - }, - { - "testName": "abortReasonTypesWorkflow: various abort reason types propagate correctly", - "runId": "wrun_01KYWTZCBM5KS9R4AC2M3Y3EYV", - "timestamp": "2026-07-31T19:38:47.036Z", - "dashboardUrl": null - }, - { - "testName": "abortFetchUncaughtWorkflow: uncaught fetch AbortError is FatalError, no retries", - "runId": "wrun_01KYWTZDB8G3YN8HJ4BSPT01C1", - "timestamp": "2026-07-31T19:38:48.052Z", - "dashboardUrl": null - }, - { - "testName": "abortFetchInFlightWorkflow: aborting cancels an in-flight fetch", - "runId": "wrun_01KYWTZEB2KSFD2EMNZGNZ4BGZ", - "timestamp": "2026-07-31T19:38:49.062Z", - "dashboardUrl": null - }, - { - "testName": "abortVoidSleepTimeoutWorkflow: documented `void sleep().then(abort)` pattern works", - "runId": "wrun_01KYWTZH94PDJPH3HYH0XJDG4K", - "timestamp": "2026-07-31T19:38:52.076Z", - "dashboardUrl": null - }, - { - "testName": "abortDeterministicBranchWorkflow: if-check takes same path on first-run and replay", - "runId": "wrun_01KYWTZM792J3PKPHFZKENNVG8", - "timestamp": "2026-07-31T19:38:55.085Z", - "dashboardUrl": null - }, - { - "testName": "abortListenerWorkflow: signal.addEventListener fires on the deserialized step signal", - "runId": "wrun_01KYWTZN6RRKXGEYV2HRW6SHQA", - "timestamp": "2026-07-31T19:38:56.093Z", - "dashboardUrl": null - }, - { - "testName": "abortThrowIfAbortedMidFlightWorkflow: throwIfAborted in a polling loop bails when abort fires", - "runId": "wrun_01KYWTZQ5G9EGEJ01G39GBC035", - "timestamp": "2026-07-31T19:38:58.102Z", - "dashboardUrl": null - }, - { - "testName": "abortDeterministicBranchFromStepWorkflow: branches stay consistent when abort comes from a step", - "runId": "wrun_01KYWTZY0REZV6A0JBA96YQKW1", - "timestamp": "2026-07-31T19:39:05.119Z", - "dashboardUrl": null - }, - { - "testName": "abortHookOrderingWorkflow [listener-first-abort-first]: addEventListener → hook.then → abort() → resumeHook", - "runId": "wrun_01KYWTZZZMQSFATHQ2JF07W8F9", - "timestamp": "2026-07-31T19:39:07.129Z", - "dashboardUrl": null - }, - { - "testName": "abortHookOrderingWorkflow [listener-first-hook-first]: addEventListener → hook.then → resumeHook → abort()", - "runId": "wrun_01KYWV0A1EKBET35VXE29YRRFX", - "timestamp": "2026-07-31T19:39:17.427Z", - "dashboardUrl": null - }, - { - "testName": "abortHookOrderingWorkflow [hook-first-abort-first]: hook.then → addEventListener → abort() → resumeHook", - "runId": "wrun_01KYWV0M2MQTPN9ESAFQWWAC91", - "timestamp": "2026-07-31T19:39:27.708Z", - "dashboardUrl": null - }, - { - "testName": "abortHookOrderingWorkflow [hook-first-hook-first]: hook.then → addEventListener → resumeHook → abort()", - "runId": "wrun_01KYWV0Y44PJA3SE5CVGDGPENP", - "timestamp": "2026-07-31T19:39:37.993Z", - "dashboardUrl": null - }, - { - "testName": "importMetaUrlWorkflow - import.meta.url is available in step bundles", - "runId": "wrun_01KYWV185GKFMWCA3GC747DWE8", - "timestamp": "2026-07-31T19:39:48.279Z", - "dashboardUrl": null - }, - { - "testName": "metadataFromHelperWorkflow - getWorkflowMetadata/getStepMetadata work from module-level helper (#1577)", - "runId": "wrun_01KYWV1952Q0DYGYYN4E59M80S", - "timestamp": "2026-07-31T19:39:49.287Z", - "dashboardUrl": null - }, - { - "testName": "resilient start: addTenWorkflow completes when run_created returns 500", - "runId": "wrun_01KYWV1A4M5PY9CRW6HW065NWK", - "timestamp": "2026-07-31T19:39:50.293Z", - "dashboardUrl": null - }, - { - "testName": "getterStepWorkflow - getter functions with \"use step\" directive", - "runId": "wrun_01KYWV1B41YKQFRRG2NTQNF6SY", - "timestamp": "2026-07-31T19:39:51.304Z", - "dashboardUrl": null - }, - { - "testName": "distributedAbortController - manual abort triggers signal", - "runId": "wrun_01KYWV1FCJ5M3ZMWDXB24C1Z84", - "timestamp": "2026-07-31T19:39:55.673Z", - "dashboardUrl": null - }, - { - "testName": "distributedAbortController - TTL expiration triggers signal", - "runId": "wrun_01KYWV1FQVXB3XDH89FW2ZWVYV", - "timestamp": "2026-07-31T19:39:56.034Z", - "dashboardUrl": null - }, - { - "testName": "distributedAbortController - reconnect to existing controller", - "runId": "wrun_01KYWV1JWW1RRCEKGZFS6R36V6", - "timestamp": "2026-07-31T19:39:59.266Z", - "dashboardUrl": null - }, - { - "testName": "start: initial attributes are seeded on run creation", - "runId": "wrun_01KYWV1M4MDRWPTX1JYJJDZM3V", - "timestamp": "2026-07-31T19:40:00.539Z", - "dashboardUrl": null - }, - { - "testName": "start: reserved-prefix initial attributes are seeded with allowReservedAttributes", - "runId": "wrun_01KYWV1N47HTHZP7ESPX0VTT5E", - "timestamp": "2026-07-31T19:40:01.550Z", - "dashboardUrl": null - }, - { - "testName": "setAttributesWorkflow: workflow-body calls append native attr_set events and merge correctly", - "runId": "wrun_01KYWV1P3SYAGGMPK5HRXQN64X", - "timestamp": "2026-07-31T19:40:02.562Z", - "dashboardUrl": null - }, - { - "testName": "setAttributesInsideStepWorkflow: step-body calls append attributed native events", - "runId": "wrun_01KYWV1Q3JW86C750J66RZJWPF", - "timestamp": "2026-07-31T19:40:03.576Z", - "dashboardUrl": null - }, - { - "testName": "fire-and-forget: void setAttributes lands without awaiting", - "runId": "wrun_01KYWV1R3BW1VYWADWJBFY12D9", - "timestamp": "2026-07-31T19:40:04.594Z", - "dashboardUrl": null - }, - { - "testName": "Promise.all of disjoint-key writes: every key lands", - "runId": "wrun_01KYWV1V1ESEQ5VPA14CQ9573Y", - "timestamp": "2026-07-31T19:40:07.603Z", - "dashboardUrl": null - }, - { - "testName": "workflow throws after awaited setAttributes: attribute still persists on the failed run", - "runId": "wrun_01KYWV1W18742A0498EFG2D27V", - "timestamp": "2026-07-31T19:40:08.658Z", - "dashboardUrl": null - }, - { - "testName": "validation DX: invalid writes throw catchable FatalErrors naming rule and limit", - "runId": "wrun_01KYWV1X20BBMKJ2HPTDM8FQDA", - "timestamp": "2026-07-31T19:40:09.669Z", - "dashboardUrl": null - } -] \ No newline at end of file From 34b8554b6e6b93a65aa03aef1d8578042e8bca22 Mon Sep 17 00:00:00 2001 From: Peter Wielander Date: Fri, 31 Jul 2026 13:33:39 -0700 Subject: [PATCH 34/35] fix(world-local): allocate slots append-only A slot names a position in the replay order, so allocation has to hand out a position no published event sits above. Handing out the lowest free position instead let a late `step_completed` drop into a hole beneath its own `step_created`/`step_started`, and every replay of that run then met a completion for a step it had not started. The book now keeps a monotonic ceiling: a reservation goes above every position the book has ever seen, and releasing one does not lower it. `run_created` takes the first slot outright rather than allocating it, since a `run_started` racing it can already have moved the book past that position. Co-Authored-By: Claude Opus 5 --- .../world-local/src/storage/events-storage.ts | 39 +++++--- .../src/storage/slot-identity.test.ts | 16 +++- .../world-local/src/storage/slots.test.ts | 88 +++++++++--------- packages/world-local/src/storage/slots.ts | 93 ++++++++++--------- 4 files changed, 133 insertions(+), 103 deletions(-) diff --git a/packages/world-local/src/storage/events-storage.ts b/packages/world-local/src/storage/events-storage.ts index 0e6c0ce39d..24d2f787c4 100644 --- a/packages/world-local/src/storage/events-storage.ts +++ b/packages/world-local/src/storage/events-storage.ts @@ -1061,12 +1061,13 @@ export function createEventsStorage( // EVENT ID: the caller's slot claim, an allocated slot, or a ULID // ============================================================ // A run's own `run_created` owns the first slot — provably, since - // nothing precedes it — so every other event allocates above it, even - // when that event is the first to arrive here. - const allocationFloor = - data.eventType === 'run_created' - ? RUN_CREATED_SLOT - : RUN_CREATED_SLOT + 1; + // nothing precedes it — so it takes that position outright instead of + // allocating one, and every other event allocates above it even when it + // is the first to arrive here. Allocation is append-only (see SlotBook), + // so `run_created` cannot get the first position by asking for the lowest + // free one: a `run_started` racing it (start() issues the creation and + // the queue send in parallel) may already have moved the book past it. + const ownsFirstSlot = data.eventType === 'run_created'; // A slot-numbered run's ids name positions in its log, so an id is // either claimed by a caller that holds the log (and is therefore // asserting the log is complete up to that position) or allocated here @@ -1130,7 +1131,13 @@ export function createEventsStorage( } } else if (slotMode) { reservedRunId = effectiveRunId; - const slot = await slots.reserve(effectiveRunId, allocationFloor); + let slot: number; + if (ownsFirstSlot) { + slot = RUN_CREATED_SLOT; + slots.claim(effectiveRunId, slot); + } else { + slot = await slots.reserve(effectiveRunId); + } reserved.add(slot); eventId = slotEventId(slot); } @@ -2691,7 +2698,12 @@ export function createEventsStorage( // to reconcile. A write whose position the *caller* claimed may not: // the claim asserts a log complete up to that position, so losing it // means that log is stale and only the caller can resolve it. - const reallocatesSlot = slotMode && params?.eventId === undefined; + // `run_created` is excluded even though it named its own position: the + // first slot is the only position it can ever occupy, so losing it means + // the run already has a creation event, and appending a second one above + // it would be worse than the duplicate the publish is reporting. + const reallocatesSlot = + slotMode && params?.eventId === undefined && !ownsFirstSlot; const slotDeadline = Date.now() + SLOT_RETRY_BUDGET_MS; let compositeKey = ''; let eventPath = ''; @@ -2712,10 +2724,11 @@ export function createEventsStorage( if (reallocatesSlot && Date.now() < slotDeadline) { // The position is someone else's — either published there or // staged for it. Record that, top the book up from disk, and try - // again one position higher rather than surfacing a conflict the - // caller cannot act on. Re-reading rather than incrementing keeps - // the log dense: every round at least one writer wins, so the - // search never runs away from the log it is filling. + // again above whatever the log has reached rather than surfacing a + // conflict the caller cannot act on. Re-reading rather than + // incrementing bounds the search: every round at least one writer + // wins, so the top of the log is never further than the number of + // writers still contending for it. const lost = slotFromId(eventId); if (lost !== undefined) { reserved.delete(lost); @@ -2725,7 +2738,7 @@ export function createEventsStorage( await new Promise((resolve) => setTimeout(resolve, slotRetryDelay(round)) ); - const slot = await slots.reserve(effectiveRunId, allocationFloor); + const slot = await slots.reserve(effectiveRunId); reservedRunId = effectiveRunId; reserved.add(slot); eventId = slotEventId(slot); diff --git a/packages/world-local/src/storage/slot-identity.test.ts b/packages/world-local/src/storage/slot-identity.test.ts index 5455e0a52f..b7c579e557 100644 --- a/packages/world-local/src/storage/slot-identity.test.ts +++ b/packages/world-local/src/storage/slot-identity.test.ts @@ -158,9 +158,11 @@ describe('numbering', () => { expect(maxSlotOf(data)).toBe(data.length); }); - it('leaves no hole behind a rejected write', async () => { - // The rejected op's slot sits below its concurrent sibling's, and a hole - // below a published event can never be filled. + it('leaves a rejected write’s position unused instead of recycling it', async () => { + // The rejected op's position sits below its concurrent sibling's, so handing + // it to the next writer would order that writer's event below one that + // already published. The hole costs a reader the density proof; the + // inversion would cost the run. const runId = await newSlotRun(); const [rejected, accepted] = await Promise.allSettled([ storage.events.create(runId, { @@ -175,7 +177,13 @@ describe('numbering', () => { expect(accepted.status).toBe('fulfilled'); await createStep(runId, 'step_b'); const slots = await slotsOf(runId); - expect([...slots].sort((a, b) => a - b)).toEqual([1, 2, 3]); + expect(slots).toHaveLength(3); + expect(slots[0]).toBe(FIRST_SLOT); + // Both concurrent writers took a position, one abandoned its own, and the + // third write went above them both. + expect(slots[2]).toBe(FIRST_SLOT + 3); + expect(slots[1]).toBeGreaterThan(slots[0]); + expect(slots[1]).toBeLessThan(slots[2]); }); }); diff --git a/packages/world-local/src/storage/slots.test.ts b/packages/world-local/src/storage/slots.test.ts index 0e60b2ff74..d7b87f9e10 100644 --- a/packages/world-local/src/storage/slots.test.ts +++ b/packages/world-local/src/storage/slots.test.ts @@ -2,7 +2,6 @@ import fs from 'node:fs/promises'; import os from 'node:os'; import path from 'node:path'; import { - FIRST_SLOT, SPEC_VERSION_CURRENT, SPEC_VERSION_SLOT_IDENTITY, slotEventId, @@ -89,9 +88,11 @@ describe('usesSlots', () => { }); describe('reserve', () => { - it('starts at the first slot for a run with no events', async () => { + it('starts above the slot the run’s own creation event owns', async () => { + // `run_created` takes the first slot outright — nothing can precede it — so + // an allocation never hands that position out. await expect(createSlotBook(basedir).reserve(RUN_ID)).resolves.toBe( - FIRST_SLOT + RUN_CREATED_SLOT + 1 ); }); @@ -100,24 +101,36 @@ describe('reserve', () => { await expect(createSlotBook(basedir).reserve(RUN_ID)).resolves.toBe(4); }); - it('fills a hole left in the persisted log', async () => { - // Density is the whole point of the scheme, so a gap that somehow exists is - // reclaimed rather than skipped over forever. + it('leaves a hole in the persisted log unfilled', async () => { + // Allocation is append-only: the free position sits below a published event, + // and an event placed there would order before one that already happened. await writeEvents(1, 3); const book = createSlotBook(basedir); - await expect(book.reserve(RUN_ID)).resolves.toBe(2); await expect(book.reserve(RUN_ID)).resolves.toBe(4); + await expect(book.reserve(RUN_ID)).resolves.toBe(5); + }); + + it('stays above published events when a lower position comes free', async () => { + // The corruption this rules out: a step completion allocating late, dropping + // into a hole, and landing below the step_started it reports on. Replay reads + // the log in slot order and cannot consume that. + const book = createSlotBook(basedir); + const abandoned = await book.reserve(RUN_ID); + const published = await book.reserve(RUN_ID); + book.observe(RUN_ID, slotEventId(published)); + book.release(RUN_ID, abandoned); + await expect(book.reserve(RUN_ID)).resolves.toBeGreaterThan(published); }); it('hands a synchronous burst distinct consecutive slots', async () => { - // The suspension flush issues every op concurrently; with a plain - // "max + 1" they would all pick the same slot and all but one would fail. + // The suspension flush issues every op concurrently; a book that only moved + // on publish would give them all the same position and fail all but one. const book = createSlotBook(basedir); const slots = await Promise.all( Array.from({ length: 20 }, () => book.reserve(RUN_ID)) ); expect([...slots].sort((a, b) => a - b)).toEqual( - Array.from({ length: 20 }, (_, index) => FIRST_SLOT + index) + Array.from({ length: 20 }, (_, index) => RUN_CREATED_SLOT + 1 + index) ); }); @@ -131,26 +144,13 @@ describe('reserve', () => { expect([...slots].sort((a, b) => a - b)).toEqual([2, 3]); }); - it('honours a floor, so a concurrent event cannot take run_created’s slot', async () => { - // start() publishes the run entity before its `run_created` event and - // issues the queue send in parallel, so the delivery's `run_started` can - // allocate while slot 1 is still in flight. - const book = createSlotBook(basedir); - await expect(book.reserve(RUN_ID, RUN_CREATED_SLOT + 1)).resolves.toBe( - RUN_CREATED_SLOT + 1 - ); - }); - - it('leaves slots below the floor allocatable', async () => { - // A floored search skips that range without looking at it, so it proves - // nothing about it — `run_created` must still find its own slot free. + it('honours a floor above where the book has reached', async () => { + // start() publishes the run entity before its `run_created` event and issues + // the queue send in parallel, so the delivery's `run_started` can allocate + // while slot 1 is still in flight. const book = createSlotBook(basedir); - await expect(book.reserve(RUN_ID, RUN_CREATED_SLOT + 1)).resolves.toBe( - RUN_CREATED_SLOT + 1 - ); - await expect(book.reserve(RUN_ID, RUN_CREATED_SLOT)).resolves.toBe( - RUN_CREATED_SLOT - ); + await expect(book.reserve(RUN_ID, 5)).resolves.toBe(5); + await expect(book.reserve(RUN_ID)).resolves.toBe(6); }); it('keeps runs independent', async () => { @@ -158,22 +158,22 @@ describe('reserve', () => { const book = createSlotBook(basedir); await expect(book.reserve(RUN_ID)).resolves.toBe(3); await expect(book.reserve('wrun_01K0000000000000000000OTHR')).resolves.toBe( - FIRST_SLOT + RUN_CREATED_SLOT + 1 ); }); }); describe('release', () => { - it('gives an abandoned interior slot to the next caller', async () => { - // A rejected op must not strand the slot below its concurrent siblings': - // that hole can never be filled once a later slot is published. + it('does not recycle an abandoned interior slot', async () => { + // The position may already sit below a sibling that published, and no + // caller can tell from here. A hole costs a reader its completeness proof; + // an inversion costs the run. const book = createSlotBook(basedir); const [first, second] = await Promise.all([ book.reserve(RUN_ID), book.reserve(RUN_ID), ]); book.release(RUN_ID, first); - await expect(book.reserve(RUN_ID)).resolves.toBe(first); await expect(book.reserve(RUN_ID)).resolves.toBe(second + 1); }); @@ -215,13 +215,15 @@ describe('claim', () => { await expect(book.reserve(RUN_ID)).resolves.toBe(3); }); - it('frees the slot again once the claim resolves', async () => { + it('stops holding back a claim that resolved', async () => { + // Released before anything allocated for the run, so no position in this + // instance was ever handed out above it and the log — which is the authority + // on what published — reaches only slot 1. Nothing can be inverted by + // seeding the book from disk alone. await writeEvents(1); const book = createSlotBook(basedir); book.claim(RUN_ID, 2); book.release(RUN_ID, 2); - // Nothing is allocating for the run yet, so the book the next caller seeds - // has to start from the log alone. await expect(book.reserve(RUN_ID)).resolves.toBe(2); }); @@ -237,27 +239,25 @@ describe('claim', () => { }); describe('observe', () => { - it('never hands out a slot claimed by the client', async () => { + it('moves allocation above a position the client claimed', async () => { const book = createSlotBook(basedir); await book.reserve(RUN_ID); book.observe(RUN_ID, slotEventId(5)); - const next = await book.reserve(RUN_ID); - expect(next).not.toBe(5); - expect(next).toBe(2); + await expect(book.reserve(RUN_ID)).resolves.toBe(6); }); it('ignores ULID event ids', async () => { const book = createSlotBook(basedir); - await book.reserve(RUN_ID); + const slot = await book.reserve(RUN_ID); book.observe(RUN_ID, 'evnt_01K5Z0000000000000000000AA'); - await expect(book.reserve(RUN_ID)).resolves.toBe(2); + await expect(book.reserve(RUN_ID)).resolves.toBe(slot + 1); }); }); describe('forget', () => { it("re-reads the log, picking up another writer's events", async () => { const book = createSlotBook(basedir); - await expect(book.reserve(RUN_ID)).resolves.toBe(FIRST_SLOT); + await expect(book.reserve(RUN_ID)).resolves.toBe(RUN_CREATED_SLOT + 1); await writeEvents(1, 2, 3); book.forget(RUN_ID); await expect(book.reserve(RUN_ID)).resolves.toBe(4); diff --git a/packages/world-local/src/storage/slots.ts b/packages/world-local/src/storage/slots.ts index 58275f3d57..25b2b8e522 100644 --- a/packages/world-local/src/storage/slots.ts +++ b/packages/world-local/src/storage/slots.ts @@ -2,21 +2,29 @@ * Slot allocation for the Local World. * * A slot-numbered run names its events by position: `evnt_…001` is the first - * event of the run, `evnt_…002` the second, with no gaps. Density is the point - * — it is what lets a reader prove its loaded log is complete — so an allocator - * must never leave a slot permanently unwritten. + * event of the run, `evnt_…002` the second. A replay reads the log in slot + * order, so the order slots are handed out in has to be an order some execution + * could have produced — which makes allocation strictly *append-only*: a slot is + * only ever handed out above every position this book has seen. + * + * Filling a hole is what that rules out, and it is worth naming why, because the + * alternative looks appealing (it keeps the log dense). A position left unwritten + * by an abandoned reservation sits below events that are already published. Hand + * it to the next caller and a `step_completed` lands below its own + * `step_started`; the replay reaches a completion for a step it has not started + * and diverges, and every later replay diverges the same way. A hole costs a + * reader the ability to prove its copy of the log is complete. An inversion + * costs the run. * * Three properties do the work: * * - Handing out a slot is a *synchronous* set operation, so concurrent * callers in one process get distinct slots with no lock. The only await is * seeding from disk, which is memoized per run. - * - An allocation always picks the lowest slot that is neither written nor - * outstanding, so a reservation that is abandoned (its create threw a - * validation error) is handed to the next caller instead of leaving an - * interior hole. This matters under fan-out: N concurrent step_completed - * writes reserve N consecutive slots, and one rejected op must not strand - * the slot below its siblings'. + * - An allocation picks the position above the highest one the book knows of, + * written or outstanding, and that ceiling never descends. A reservation that + * is abandoned (its create threw a validation error) leaves its position + * unused rather than being recycled below a sibling that already published. * - The event publish is `writeExclusive`, which is the authority. The book is * a hint: when it turns out to be stale (another process wrote the slot), * the publish fails and the caller is told so, rather than a duplicate being @@ -43,8 +51,12 @@ interface RunSlots { written: Set; /** Slots handed out whose publish has not resolved yet. */ outstanding: Set; - /** Lowest slot that might still be free; never decreases except on release. */ - searchFrom: number; + /** + * The highest position this book has ever seen written, claimed or handed out. + * Allocation goes above it and it never descends, which is what keeps a + * released position from being recycled below events already published. + */ + ceiling: number; } export interface SlotBook { @@ -58,16 +70,15 @@ export interface SlotBook { */ usesSlots(runId: string): Promise; /** - * Reserves the lowest free slot of `runId`, at or above `minSlot`. Distinct - * for every concurrent caller; the publish still has to prove the slot was - * actually free. + * Reserves the position above every one this book knows of for `runId`, and at + * or above `minSlot`. Distinct for every concurrent caller; the publish still + * has to prove the position was actually free. * - * `minSlot` is how the run's first slot is kept for its own `run_created`: - * that event's slot needs no allocation, and it may not be on disk yet when a - * concurrent `run_started` allocates (start() issues the creation and the - * queue send in parallel, and the run entity is published before its event). - * Slots below `minSlot` stay allocatable for a later caller, so holding one - * back leaves the log dense. + * `minSlot` defaults to the position above the run's first slot, which is + * reserved for its own `run_created`: that event needs no allocation, and it + * may not be on disk yet when a concurrent `run_started` allocates (start() + * issues the creation and the queue send in parallel, and the run entity is + * published before its event). */ reserve(runId: string, minSlot?: number): Promise; /** @@ -90,8 +101,10 @@ export interface SlotBook { */ isWritten(runId: string, slot: number): Promise; /** - * Returns a reserved or claimed slot that was never published, so the next - * caller takes it instead of it becoming a hole. + * Forgets a reserved or claimed slot whose publish is never going to happen, + * so nothing waits on it. The position itself is not handed out again: it may + * already sit below a sibling that published, and recycling it there would put + * a later event below an earlier one. */ release(runId: string, slot: number): void; /** Records a published event id, so it is never handed out again. */ @@ -147,10 +160,11 @@ export function createSlotBook(basedir: string, tag?: string): SlotBook { written.add(slot); } } + const outstanding = new Set(claims.get(runId)); const book: RunSlots = { written, - outstanding: new Set(claims.get(runId)), - searchFrom: FIRST_SLOT, + outstanding, + ceiling: Math.max(FIRST_SLOT - 1, ...written, ...outstanding), }; books.set(runId, book); return book; @@ -186,17 +200,9 @@ export function createSlotBook(basedir: string, tag?: string): SlotBook { } function take(book: RunSlots, minSlot: number): number { - let slot = Math.max(book.searchFrom, minSlot); - while (book.written.has(slot) || book.outstanding.has(slot)) { - slot += 1; - } + const slot = Math.max(book.ceiling + 1, minSlot); book.outstanding.add(slot); - if (minSlot <= book.searchFrom) { - // Only an unfloored search proves everything below the slot it landed on - // is taken. A floored one skipped that range without looking, and those - // slots are still free for a caller that may take them. - book.searchFrom = slot; - } + book.ceiling = slot; return slot; } @@ -216,7 +222,7 @@ export function createSlotBook(basedir: string, tag?: string): SlotBook { return mode; }, - async reserve(runId, minSlot = FIRST_SLOT) { + async reserve(runId, minSlot = RUN_CREATED_SLOT + 1) { const opened = open(runId); // Awaiting a book that is already in hand would yield to the microtask // queue and let a concurrent caller take the same slot. @@ -230,7 +236,11 @@ export function createSlotBook(basedir: string, tag?: string): SlotBook { } else { claims.set(runId, new Set([slot])); } - books.get(runId)?.outstanding.add(slot); + const book = books.get(runId); + if (book) { + book.outstanding.add(slot); + book.ceiling = Math.max(book.ceiling, slot); + } }, async isWritten(runId, slot) { @@ -244,10 +254,10 @@ export function createSlotBook(basedir: string, tag?: string): SlotBook { if (!book) { return; } + // The ceiling stays where it is: this position may already sit below one a + // sibling published, and handing it out again would order a later event + // before an earlier one. book.outstanding.delete(slot); - if (slot < book.searchFrom) { - book.searchFrom = slot; - } }, observe(runId, eventId) { @@ -264,6 +274,7 @@ export function createSlotBook(basedir: string, tag?: string): SlotBook { } book.written.add(slot); book.outstanding.delete(slot); + book.ceiling = Math.max(book.ceiling, slot); }, async refresh(runId) { @@ -277,11 +288,9 @@ export function createSlotBook(basedir: string, tag?: string): SlotBook { if (slot !== undefined) { book.written.add(slot); book.outstanding.delete(slot); + book.ceiling = Math.max(book.ceiling, slot); } } - // Positions released while this scan ran may sit below where the search - // had reached, and they are free again. - book.searchFrom = FIRST_SLOT; }, forget(runId) { From 61e7dc87e6682c50c0c7b836727305cd463b58ba Mon Sep 17 00:00:00 2001 From: Peter Wielander Date: Fri, 31 Jul 2026 13:39:43 -0700 Subject: [PATCH 35/35] chore: sort imports --- packages/core/src/runtime.test.ts | 2 +- packages/core/src/runtime.ts | 2 +- packages/world-local/src/storage/events-storage.ts | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/packages/core/src/runtime.test.ts b/packages/core/src/runtime.test.ts index 92dc696419..ff4c0830c9 100644 --- a/packages/core/src/runtime.test.ts +++ b/packages/core/src/runtime.test.ts @@ -15,7 +15,6 @@ import { import { ulid } from 'ulid'; import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest'; import { registerStepFunction } from './private.js'; -import { getWorkflowMetadata } from './step/get-workflow-metadata.js'; import { REPLAY_DIVERGENCE_MAX_RETRIES } from './runtime/constants.js'; import { setWorld } from './runtime/world.js'; import { workflowEntrypoint } from './runtime.js'; @@ -23,6 +22,7 @@ import { dehydrateStepReturnValue, dehydrateWorkflowArguments, } from './serialization.js'; +import { getWorkflowMetadata } from './step/get-workflow-metadata.js'; // Capture every promise handed to `waitUntil` so tests can assert that // progress-critical sends are never registered on a detached, unconsumed diff --git a/packages/core/src/runtime.ts b/packages/core/src/runtime.ts index 3e4915b678..5e711b9182 100644 --- a/packages/core/src/runtime.ts +++ b/packages/core/src/runtime.ts @@ -53,7 +53,6 @@ import { appendUniqueEvents, type EventCreator, eventCreateFenceFor, - stepClaimFence, getQueueOverhead, getWorkflowQueueName, handleHealthCheckMessage, @@ -63,6 +62,7 @@ import { parseHealthCheckPayload, queueMessage, requiresFreshReplay, + stepClaimFence, toMutableEventLog, withEventCreateFence, withHealthCheck, diff --git a/packages/world-local/src/storage/events-storage.ts b/packages/world-local/src/storage/events-storage.ts index 24d2f787c4..c84ffbf7c7 100644 --- a/packages/world-local/src/storage/events-storage.ts +++ b/packages/world-local/src/storage/events-storage.ts @@ -35,9 +35,9 @@ import { isTerminalStepStatus, isTerminalWorkflowRunStatus, requiresNewerWorld, + SLOT_RETRY_BUDGET_MS, SPEC_VERSION_CURRENT, SPEC_VERSION_MAX_SUPPORTED, - SLOT_RETRY_BUDGET_MS, StepSchema, slotEventId, slotFromId,