From 0976f3c707ef45af5e28cc07fbbd788f25a5f8c0 Mon Sep 17 00:00:00 2001 From: Nathan Colosimo <110621881+NathanColosimo@users.noreply.github.com> Date: Thu, 13 Aug 2026 20:15:08 -0700 Subject: [PATCH 01/16] Prepare replay payloads during event streaming --- .../prepare-streamed-replay-payloads.md | 5 + .../core/src/replay-payload-cache.test.ts | 95 +++- packages/core/src/replay-payload-cache.ts | 417 +++++++++++------- packages/core/src/runtime.ts | 135 ++++-- packages/core/src/runtime/helpers.test.ts | 12 + packages/core/src/runtime/helpers.ts | 10 +- .../runtime/quickjs-partial-preload.test.ts | 1 + .../core/src/step-delivery-ordering.test.ts | 2 +- packages/core/src/step.ts | 65 +-- packages/core/src/workflow/hook.ts | 59 ++- 10 files changed, 552 insertions(+), 249 deletions(-) create mode 100644 .changeset/prepare-streamed-replay-payloads.md diff --git a/.changeset/prepare-streamed-replay-payloads.md b/.changeset/prepare-streamed-replay-payloads.md new file mode 100644 index 0000000000..9678641541 --- /dev/null +++ b/.changeset/prepare-streamed-replay-payloads.md @@ -0,0 +1,5 @@ +--- +"@workflow/core": patch +--- + +Prepare replay payloads as event frames arrive and reuse immutable primitive values across fresh workflow VMs. diff --git a/packages/core/src/replay-payload-cache.test.ts b/packages/core/src/replay-payload-cache.test.ts index 7f89f43cd4..dc7d77ddf8 100644 --- a/packages/core/src/replay-payload-cache.test.ts +++ b/packages/core/src/replay-payload-cache.test.ts @@ -133,6 +133,47 @@ describe('ReplayPayloadCache', () => { allSettled.mockRestore(); }); + it('prepares streamed events synchronously inside the decoder callback', async () => { + const payload = new Uint8Array([1]); + const order: string[] = []; + const preparer = vi.fn((value) => { + order.push('prepare'); + return { data: value }; + }); + const cache = new ReplayPayloadCache(undefined, preparer); + const [event] = makeEvents([payload]); + + const preparation = cache.observeEvent(event, () => order.push('start')); + expect(preparation).toBeDefined(); + expect(preparer).toHaveBeenCalledOnce(); + expect(order).toEqual(['start', 'prepare']); + + // Re-observing a cache hit does not move the preparation-span boundary. + cache.observeEvent(event, () => order.push('cached-start')); + expect(order).toEqual(['start', 'prepare']); + + await expect(preparation).resolves.toEqual({ data: payload }); + }); + + it('prepares queued stream events as soon as the run key resolves', async () => { + const payload = new Uint8Array([1]); + const preparer = vi.fn((value) => ({ data: value })); + let resolveKey!: (key: undefined) => void; + const key = new Promise((resolve) => { + resolveKey = resolve; + }); + const cache = new ReplayPayloadCache(key, preparer); + const [event] = makeEvents([payload]); + + const preparation = cache.observeEvent(event); + expect(preparation).toBeDefined(); + expect(preparer).not.toHaveBeenCalled(); + + resolveKey(undefined); + await expect(preparation).resolves.toEqual({ data: payload }); + expect(preparer).toHaveBeenCalledOnce(); + }); + it('caches real decrypt/decompress output but revives fresh objects', async () => { const key = await importKey(new Uint8Array(32).fill(7)); const serialized = await dehydrateStepReturnValue( @@ -148,6 +189,10 @@ describe('ReplayPayloadCache', () => { const preparer = vi.fn(decodePayload); const cache = new ReplayPayloadCache(key, preparer); + const directPreparation = prepareReplayPayload(serialized, key); + expect(directPreparation).not.toBeInstanceOf(Promise); + await directPreparation; + const prepared = await cache.prepareEventPayload( 'evnt_encrypted', 'result', @@ -222,12 +267,34 @@ describe('ReplayPayloadCache', () => { const cache = new ReplayPayloadCache(undefined); const hydrate = vi.fn().mockResolvedValue(value); - expect(await cache.getStepResult('evnt_result', hydrate)).toBe(value); - expect(await cache.getStepResult('evnt_result', hydrate)).toBe(value); + expect( + await cache.getPrimitiveValue('evnt_result', 'result', hydrate) + ).toBe(value); + expect( + await cache.getPrimitiveValue('evnt_result', 'result', hydrate) + ).toBe(value); expect(hydrate).toHaveBeenCalledOnce(); } }); + it('isolates primitive values by event payload field', async () => { + const cache = new ReplayPayloadCache(undefined); + const result = vi.fn().mockResolvedValue('result'); + const error = vi.fn().mockResolvedValue('error'); + + await expect( + cache.getPrimitiveValue('evnt_shared', 'result', result) + ).resolves.toBe('result'); + await expect( + cache.getPrimitiveValue('evnt_shared', 'error', error) + ).resolves.toBe('error'); + await expect( + cache.getPrimitiveValue('evnt_shared', 'result', result) + ).resolves.toBe('result'); + expect(result).toHaveBeenCalledOnce(); + expect(error).toHaveBeenCalledOnce(); + }); + it('rehydrates mutable and oversized step results', async () => { const oversized = 'x'.repeat(4097); for (const value of [{ count: 0 }, oversized]) { @@ -238,8 +305,16 @@ describe('ReplayPayloadCache', () => { typeof value === 'object' ? { ...value } : value ); - const first = await cache.getStepResult('evnt_result', hydrate); - const second = await cache.getStepResult('evnt_result', hydrate); + const first = await cache.getPrimitiveValue( + 'evnt_result', + 'result', + hydrate + ); + const second = await cache.getPrimitiveValue( + 'evnt_result', + 'result', + hydrate + ); expect(hydrate).toHaveBeenCalledTimes(2); if (typeof value === 'object') expect(second).not.toBe(first); } @@ -252,12 +327,12 @@ describe('ReplayPayloadCache', () => { .mockRejectedValueOnce(new Error('boom')) .mockResolvedValueOnce('ok'); - await expect(cache.getStepResult('evnt_result', hydrate)).rejects.toThrow( - 'boom' - ); - await expect(cache.getStepResult('evnt_result', hydrate)).resolves.toBe( - 'ok' - ); + await expect( + cache.getPrimitiveValue('evnt_result', 'result', hydrate) + ).rejects.toThrow('boom'); + await expect( + cache.getPrimitiveValue('evnt_result', 'result', hydrate) + ).resolves.toBe('ok'); expect(hydrate).toHaveBeenCalledTimes(2); }); }); diff --git a/packages/core/src/replay-payload-cache.ts b/packages/core/src/replay-payload-cache.ts index 0922171662..5d3748c883 100644 --- a/packages/core/src/replay-payload-cache.ts +++ b/packages/core/src/replay-payload-cache.ts @@ -7,8 +7,10 @@ import { } from './serialization/payload.js'; import { recordCompression } from './serialization/telemetry.js'; -const MAX_MEMOIZED_PRIMITIVE_LENGTH = 4096; -type ReplayPayloadField = 'result' | 'error' | 'payload'; +type ReplayPayloadPreparer = ( + data: Uint8Array, + key: DecryptionKey | undefined +) => PreparedReplayPayload | Promise; /** Copy a view only when retaining it would also retain unrelated bytes. */ function compactOwnedBytes(data: Uint8Array): Uint8Array { @@ -17,208 +19,321 @@ function compactOwnedBytes(data: Uint8Array): Uint8Array { : new Uint8Array(data); } -function isMemoizablePrimitive(value: unknown): boolean { - if (value === null) return true; - const type = typeof value; - if (type === 'object' || type === 'function') return false; - if (type === 'string') { - return (value as string).length <= MAX_MEMOIZED_PRIMITIVE_LENGTH; - } - if (type === 'bigint') { - return (value as bigint).toString().length <= MAX_MEMOIZED_PRIMITIVE_LENGTH; - } - return true; +async function prepareReplayPayload( + data: Uint8Array, + key: DecryptionKey | undefined +): Promise { + const compressionStats: CompressionStats = {}; + const prepared = await decodePayload(data, key, compressionStats); + await recordCompression(compressionStats, 'deserialize'); + return { data: compactOwnedBytes(prepared) }; +} + +type ReplayPayloadField = 'result' | 'error' | 'payload'; + +type KeyState = + | { state: 'pending'; promise: Promise } + | { state: 'ready'; value: DecryptionKey | undefined } + | { state: 'failed'; error: unknown }; + +type Preparation = + | { state: 'waiting'; value: Uint8Array } + | { state: 'ready'; value: PreparedReplayPayload } + | { state: 'pending'; promise: Promise } + | { state: 'failed'; error: unknown }; + +function isPrimitive(value: unknown): boolean { + return value === null || !['object', 'function'].includes(typeof value); } /** * Invocation-scoped cache for replay payload hydration. * - * A workflow invocation may replay the same event log through several fresh - * VMs. This cache keeps the VM-independent decrypt/decompress result across - * those replays. Deserialization still runs against each VM's globals so every - * replay receives fresh object graphs and correctly revived Workflow objects. - * - * Successful prepared plaintext remains resident for the invocation lifetime. - * Its memory cost is the sum of decrypted and decompressed payload sizes, but - * it never crosses workflow runs or queue deliveries. + * The cache retains VM-independent decrypt/decompress output across fresh VMs. + * Deserialization still runs against each VM's globals so object graphs and + * Workflow objects remain realm-local. Primitive final values are safe to + * share and skip that repeated deserialization entirely. */ export class ReplayPayloadCache { - private readonly preparedPayloads = new Map< - string, + private readonly preparations = new Map(); + private readonly pendingPreparations = new Set< Promise >(); - private readonly primitiveStepResults = new Map(); + private readonly primitiveValues = new Map(); private nextUnscannedEventIndex = 0; - constructor( - private readonly encryptionKey: DecryptionKey | undefined, - private readonly preparer: typeof decodePayload = decodePayload - ) {} + private constructor( + private key: KeyState, + private readonly preparer: ReplayPayloadPreparer = prepareReplayPayload + ) { + if (key.state === 'pending') { + void key.promise.then( + (value) => this.resolveKey(value), + (error) => this.rejectKey(error) + ); + } + } + + static unencrypted( + preparer: ReplayPayloadPreparer = prepareReplayPayload + ): ReplayPayloadCache { + return new ReplayPayloadCache( + { state: 'ready', value: undefined }, + preparer + ); + } + + static withKey( + key: DecryptionKey, + preparer: ReplayPayloadPreparer = prepareReplayPayload + ): ReplayPayloadCache { + return new ReplayPayloadCache({ state: 'ready', value: key }, preparer); + } + + static waitingForKey( + key: Promise, + preparer: ReplayPayloadPreparer = prepareReplayPayload + ): ReplayPayloadCache { + return new ReplayPayloadCache({ state: 'pending', promise: key }, preparer); + } + + /** Start preparing an event as soon as its frame has been decoded. */ + observeEvent(event: Event, onPreparationStart?: () => void): void { + switch (event.eventType) { + case 'run_created': + this.start( + this.workflowInputKey(event.runId), + event.eventData.input, + onPreparationStart + ); + break; + case 'run_started': + this.start( + this.workflowInputKey(event.runId), + event.eventData?.input, + onPreparationStart + ); + break; + case 'step_completed': + this.start( + this.eventPayloadKey(event.eventId, 'result'), + event.eventData?.result, + onPreparationStart + ); + break; + case 'step_failed': + this.start( + this.eventPayloadKey(event.eventId, 'error'), + event.eventData?.error, + onPreparationStart + ); + break; + case 'hook_received': + this.start( + this.eventPayloadKey(event.eventId, 'payload'), + event.eventData?.payload, + onPreparationStart + ); + } + } /** - * Start every missing binary preparation before workflow execution. Failures - * are intentionally retained: the ordered event consumer must observe the - * original rejection before that entry becomes retryable. + * Start every preparation not already observed from the event stream. + * Returns a Promise only when a sealed or portable codec is still running. */ - async prewarm(workflowRun: WorkflowRun, events: Event[]): Promise { - const preparations: Promise[] = []; - const start = (cacheKey: string, value: unknown): void => { - // Legacy flattened values may be mutated by devalue's unflatten and are - // therefore prepared only by their eventual consumer, never cached. - if (!(value instanceof Uint8Array)) return; - - // Each replay scans the full event log, so awaiting cached promises here - // would add O(N^2) promise reactions over an N-step invocation. Only wait - // for preparations first discovered by this prewarm pass. - if (this.preparedPayloads.has(cacheKey)) return; - preparations.push(this.ensurePreparation(cacheKey, value)); - }; - - start(this.workflowInputKey(workflowRun.runId), workflowRun.input); - // This cache is scoped to one invocation. Incremental loads and write - // response deltas only ever append, so the scanned length locates the - // events added since the previous replay. A reload that can insert events - // BELOW that length — a stale-snapshot restart replacing the log with a - // corrected one — must call `resetScan()` first, or the inserted events are - // never scanned. Prepared entries stay valid across that: they are keyed by - // event id, not by position. + prewarm(workflowRun: WorkflowRun, events: Event[]): void | Promise { + this.start( + this.workflowInputKey(workflowRun.runId), + workflowRun.input + ); for ( let index = this.nextUnscannedEventIndex; index < events.length; index++ ) { - const event = events[index]; - switch (event.eventType) { - case 'step_completed': - start( - this.eventPayloadKey(event.eventId, 'result'), - event.eventData?.result - ); - break; - case 'step_failed': - start( - this.eventPayloadKey(event.eventId, 'error'), - event.eventData?.error - ); - break; - case 'hook_received': - start( - this.eventPayloadKey(event.eventId, 'payload'), - event.eventData?.payload - ); - break; - } + this.observeEvent(events[index]); } this.nextUnscannedEventIndex = events.length; - - // Prewarming is speculative and must not fail replay before the matching - // event is consumed. allSettled also attaches rejection handlers eagerly. - await Promise.allSettled(preparations); + return this.waitForPending(); } - /** - * Forget how much of the event log has been scanned, so the next - * {@link prewarm} walks it from the start again. - * - * Required before a replay whose event log was reloaded rather than extended: - * a corrected log inserts the events the previous load was missing, which - * shifts every later position, so a positional resume would skip exactly the - * events the reload was for. Already-prepared payloads are kept — they are - * keyed by event id, so re-scanning re-observes them for free. - */ + /** A corrected reload may insert events before the previous scan position. */ resetScan(): void { this.nextUnscannedEventIndex = 0; } - /** Return the workflow input after shared host-side preparation. */ prepareWorkflowInput( workflowRun: WorkflowRun - ): Promise { - return this.consumePreparation( + ): PreparedReplayPayload | Promise { + return this.consume( this.workflowInputKey(workflowRun.runId), workflowRun.input ); } - /** - * Return an event payload after shared host-side preparation. A rejected - * preparation is evicted only after this ordered consumer requests it, so a - * later replay can retry without hiding the original failure. - */ prepareEventPayload( eventId: string, field: ReplayPayloadField, value: unknown - ): Promise { - return this.consumePreparation(this.eventPayloadKey(eventId, field), value); + ): PreparedReplayPayload | Promise { + return this.consume(this.eventPayloadKey(eventId, field), value); } - /** - * Reuse final step values only when sharing them across VMs is unobservable. - * Objects and large strings/bigints always run `hydrate` again, producing a - * fresh VM-specific value from the separately cached prepared payload. - */ - async getStepResult( + getEventValue( eventId: string, - hydrate: () => Promise - ): Promise { - if (this.primitiveStepResults.has(eventId)) { - return this.primitiveStepResults.get(eventId); + field: ReplayPayloadField, + serializedValue: unknown, + hydrate: (prepared: PreparedReplayPayload) => unknown | Promise + ): unknown | Promise { + const cacheKey = this.eventPayloadKey(eventId, field); + if (this.primitiveValues.has(cacheKey)) { + return this.primitiveValues.get(cacheKey); } - const value = await hydrate(); - if (isMemoizablePrimitive(value)) { - this.primitiveStepResults.set(eventId, value); - } + const prepared = this.prepareEventPayload( + eventId, + field, + serializedValue + ); + const hydrateAndCache = (payload: PreparedReplayPayload) => { + const hydrated = hydrate(payload); + return hydrated instanceof Promise + ? hydrated.then((value) => this.cachePrimitive(cacheKey, value)) + : this.cachePrimitive(cacheKey, hydrated); + }; + return prepared instanceof Promise + ? prepared.then(hydrateAndCache) + : hydrateAndCache(prepared); + } + + private cachePrimitive(cacheKey: string, value: unknown): unknown { + if (isPrimitive(value)) this.primitiveValues.set(cacheKey, value); return value; } - /** - * Consumer-facing lookup. Binary payloads share preparation; legacy values - * bypass the cache because their flattened representation may be mutated. - */ - private consumePreparation( + private start( cacheKey: string, - value: unknown - ): Promise { - if (!(value instanceof Uint8Array)) { - return Promise.resolve({ data: value }); + value: unknown, + onPreparationStart?: () => void + ): void { + if (!(value instanceof Uint8Array) || this.preparations.has(cacheKey)) { + return; + } + + onPreparationStart?.(); + switch (this.key.state) { + case 'pending': + this.preparations.set(cacheKey, { state: 'waiting', value }); + break; + case 'ready': + this.prepare(cacheKey, value, this.key.value); + break; + case 'failed': + this.preparations.set(cacheKey, { + state: 'failed', + error: this.key.error, + }); + break; } + } - const preparation = this.ensurePreparation(cacheKey, value); - void preparation.catch(() => { - if (this.preparedPayloads.get(cacheKey) === preparation) { - this.preparedPayloads.delete(cacheKey); + private prepare( + cacheKey: string, + value: Uint8Array, + key: DecryptionKey | undefined + ): void { + try { + const result = this.preparer(value, key); + if (!(result instanceof Promise)) { + this.preparations.set(cacheKey, { state: 'ready', value: result }); + return; } - }); - return preparation; + + this.preparations.set(cacheKey, { state: 'pending', promise: result }); + this.pendingPreparations.add(result); + void result.then( + (prepared) => { + this.pendingPreparations.delete(result); + const current = this.preparations.get(cacheKey); + if (current?.state === 'pending' && current.promise === result) { + this.preparations.set(cacheKey, { + state: 'ready', + value: prepared, + }); + } + }, + (error) => { + this.pendingPreparations.delete(result); + const current = this.preparations.get(cacheKey); + if (current?.state === 'pending' && current.promise === result) { + this.preparations.set(cacheKey, { state: 'failed', error }); + } + } + ); + } catch (error) { + this.preparations.set(cacheKey, { state: 'failed', error }); + } } - /** Start preparation once and share the exact in-flight promise. */ - private ensurePreparation( + private consume( cacheKey: string, - value: Uint8Array - ): Promise { - const cached = this.preparedPayloads.get(cacheKey); - if (cached) return cached; - - const preparation = this.runPreparation(value); - this.preparedPayloads.set(cacheKey, preparation); - return preparation; - } - - /** Compact prepared bytes before retaining them for the invocation. */ - private async runPreparation( - value: Uint8Array - ): Promise { - const compressionStats: CompressionStats = {}; - const prepared = await this.preparer( - value, - this.encryptionKey, - compressionStats - ); - await recordCompression(compressionStats, 'deserialize'); - return { data: compactOwnedBytes(prepared) }; + value: unknown + ): PreparedReplayPayload | Promise { + if (!(value instanceof Uint8Array)) return { data: value }; + + this.start(cacheKey, value); + const preparation = this.preparations.get(cacheKey); + if (!preparation) { + throw new Error(`Replay payload preparation was not started: ${cacheKey}`); + } + + switch (preparation.state) { + case 'ready': + return preparation.value; + case 'pending': + return preparation.promise; + case 'failed': + this.preparations.delete(cacheKey); + throw preparation.error; + case 'waiting': + if (this.key.state !== 'pending') { + throw new Error(`Replay payload key was not resolved: ${cacheKey}`); + } + return this.key.promise.then(() => this.consume(cacheKey, value)); + } + } + + private waitForPending(): void | Promise { + if ( + this.key.state === 'pending' && + [...this.preparations.values()].some( + (preparation) => preparation.state === 'waiting' + ) + ) { + return this.key.promise.then(() => this.waitForPending()); + } + if (this.pendingPreparations.size === 0) return; + return Promise.allSettled([...this.pendingPreparations]).then(() => {}); + } + + private resolveKey(value: DecryptionKey | undefined): void { + if (this.key.state !== 'pending') return; + this.key = { state: 'ready', value }; + for (const [cacheKey, preparation] of this.preparations) { + if (preparation.state === 'waiting') { + this.prepare(cacheKey, preparation.value, value); + } + } + } + + private rejectKey(error: unknown): void { + if (this.key.state !== 'pending') return; + this.key = { state: 'failed', error }; + for (const [cacheKey, preparation] of this.preparations) { + if (preparation.state === 'waiting') { + this.preparations.set(cacheKey, { state: 'failed', error }); + } + } } private workflowInputKey(runId: string): string { diff --git a/packages/core/src/runtime.ts b/packages/core/src/runtime.ts index 11667984f4..e11965cf26 100644 --- a/packages/core/src/runtime.ts +++ b/packages/core/src/runtime.ts @@ -14,7 +14,7 @@ import { WorkflowRuntimeError, WorkflowWorldError, } from '@workflow/errors'; -import { once, setWorkflowBasePath } from '@workflow/utils'; +import { once, setWorkflowBasePath, withResolvers } from '@workflow/utils'; import { parseWorkflowName, workflowDisplayName, @@ -541,6 +541,32 @@ function appendEventLog(log: LoadedEventLog, appended: LoadedEventLog): void { log.cursor = appended.cursor ?? log.cursor; } +function replayEventDeploymentId(event: Event): string | undefined { + if (event.eventType === 'run_created' || event.eventType === 'run_started') { + return event.eventData?.deploymentId; + } + return undefined; +} + +function createReplayEventObserver({ + runId, + cache, + resolveKey, +}: { + runId: string; + cache: ReplayPayloadCache; + resolveKey: ( + runOrId: WorkflowRun | string, + context?: Record + ) => void; +}): (event: Event) => void { + return (event) => { + const deploymentId = replayEventDeploymentId(event); + if (deploymentId) resolveKey(runId, { deploymentId }); + cache.observeEvent(event); + }; +} + /** * The whole retention predicate: keep the session only for a pure step * boundary (every suspension item is a step — any other item type, present @@ -1014,6 +1040,43 @@ export function workflowEntrypoint( // describe the next load exactly. let eventLog: ReplayEventLog = { type: 'loadAll' }; + // Resolve the run-scoped key as soon as the deployment id is + // known. On a normal replay that is the streamed run_created + // frame; resilient start and turbo already carry it in the + // queue payload. This starts key resolution and payload + // preparation before the remainder of the event log arrives, + // without guessing the key for a cross-deployment run. + const { + promise: replayKeySource, + resolve: resolveReplayKeySource, + } = withResolvers<{ + runOrId: WorkflowRun | string; + context?: Record; + }>(); + const resolveReplayKey = ( + runOrId: WorkflowRun | string, + context?: Record + ): void => { + resolveReplayKeySource({ runOrId, context }); + }; + const encryptionKeyPromise = replayKeySource.then( + ({ runOrId, context }) => + memoizeEncryptionKey(world, runOrId, context)() + ); + const replayPayloadCache = new ReplayPayloadCache( + encryptionKeyPromise + ); + const observeReplayEvent = createReplayEventObserver({ + runId, + cache: replayPayloadCache, + resolveKey: resolveReplayKey, + }); + if (runInput?.deploymentId) { + resolveReplayKey(runId, { + deploymentId: runInput.deploymentId, + }); + } + // Shared state: set by either the background step path // or the run_started setup below. let workflowRun: WorkflowRun | undefined; @@ -1985,6 +2048,7 @@ export function workflowEntrypoint( resumeId: hookResumeInput.resumeId, resumePayloadDigest: hookResumeInput.payloadDigest, preloadEvents: true, + onEvent: observeReplayEvent, } ); hookEnsured = true; @@ -2259,6 +2323,7 @@ export function workflowEntrypoint( }); const result = await createEvent(runStartedEvent, { requestId, + onEvent: observeReplayEvent, }); workflowRun = result.run; maxEventsLimit = clampMaxEvents(result.maxEvents); @@ -2543,31 +2608,18 @@ export function workflowEntrypoint( // do we fall back to reloading the complete log. if (eventLog.type !== 'loadAll' && ensuredEvent) { insertEventByEventId(eventLog.events, ensuredEvent); + observeReplayEvent(ensuredEvent); } else { eventLog = { type: 'loadAll' }; } } // end else (re-ensure needed) } - // Resolve the encryption key for this run's deployment. - // Used eagerly here since both workflow execution (input - // hydration / hook payload decryption) and the run_failed - // dehydrate path below need it. Memoized accessor: first - // call triggers the actual fetch / HKDF derivation, - // subsequent calls await the cached promise. - const getEncryptionKey = memoizeEncryptionKey( - world, - workflowRun - ); - const encryptionKey = await getEncryptionKey(); - - // Invocation-scoped cache of VM-independent prepared payloads - // and immutable final values. It survives the fresh workflow - // VM created by each inline replay, but never crosses runs or - // queue deliveries. - const replayPayloadCache = new ReplayPayloadCache( - encryptionKey - ); + // Worlds that do not implement streamed observation still + // resolve from the materialized run. This is also the final + // cross-deployment-safe source of truth. + resolveReplayKey(workflowRun); + const encryptionKey = await encryptionKeyPromise; // The live VM parked at the previous boundary, when the // retention decision kept it. null → this iteration cold- @@ -2662,7 +2714,11 @@ export function workflowEntrypoint( if (eventLog.type === 'loadAfter') { appendEventLog( eventLog, - await loadWorkflowRunEvents(runId, eventLog.cursor) + await loadWorkflowRunEvents( + runId, + eventLog.cursor, + observeReplayEvent + ) ); eventLog = { ...eventLog, type: 'ready' }; } @@ -2719,7 +2775,8 @@ export function workflowEntrypoint( runId, eventLog.type === 'loadAfter' ? eventLog.cursor - : undefined + : undefined, + observeReplayEvent ); if (eventLog.type === 'loadAfter') { appendEventLog(eventLog, page); @@ -2839,7 +2896,8 @@ export function workflowEntrypoint( if (eventLog.cursor) { const page = await loadWorkflowRunEvents( runId, - eventLog.cursor + eventLog.cursor, + observeReplayEvent ); const completedWaitIdsAfterCursor = new Set( page.events @@ -2857,13 +2915,21 @@ export function workflowEntrypoint( appendEventLog(eventLog, page); } else { eventLog = { - ...(await loadWorkflowRunEvents(runId)), + ...(await loadWorkflowRunEvents( + runId, + undefined, + observeReplayEvent + )), type: 'ready', }; } } else { eventLog = { - ...(await loadWorkflowRunEvents(runId)), + ...(await loadWorkflowRunEvents( + runId, + undefined, + observeReplayEvent + )), type: 'ready', }; } @@ -2946,15 +3012,20 @@ export function workflowEntrypoint( if (resumeTracking) { resumeTracking.replayStartedAtMs ??= replayStart; } - // Start every missing decrypt/decompress operation up - // front (already-prepared payloads are skipped). Web - // Crypto work overlaps VM setup on the replay path and - // the appended events' consumption on the resume path; - // consumers still deserialize and resolve in event order. + // Finish scheduling every missing decrypt/decompress + // operation (stream-observed payloads are already in + // flight). Preparation overlaps VM setup on replay and + // appended-event consumption on resume; consumers still + // deserialize and resolve in event order. + const replayEvents = eventLog.events; const payloadPrewarm = replayPayloadCache.prewarm( workflowRun, - eventLog.events + replayEvents ); + // Consumers await their own prepared payloads in event + // order. Do not delay a suspension on speculative work + // for payloads this replay never touched. + void payloadPrewarm.catch(() => {}); let workflowResult: WorkflowResumeResult = retainedSession ? await resumeWorkflow(retainedSession, eventLog.events) : { type: 'replay' }; @@ -2974,8 +3045,6 @@ export function workflowEntrypoint( worldCapabilities: world.capabilities, }); } - await payloadPrewarm; - if (workflowResult.type === 'suspended') { // Park the live session; the suspension catch below // makes the one retention decision — keep it for the diff --git a/packages/core/src/runtime/helpers.test.ts b/packages/core/src/runtime/helpers.test.ts index 0ce474ef07..0854f5d381 100644 --- a/packages/core/src/runtime/helpers.test.ts +++ b/packages/core/src/runtime/helpers.test.ts @@ -971,6 +971,18 @@ describe('memoizeEncryptionKey', () => { expect(spy).toHaveBeenCalledTimes(1); }); + it('passes deployment context when resolving before the run is materialized', async () => { + const spy = vi.fn().mockResolvedValue(MATERIAL); + const getKey = memoizeEncryptionKey(worldWithKey(spy), 'wrun_1', { + deploymentId: 'dpl_streamed', + }); + + await getKey(); + expect(spy).toHaveBeenCalledWith('wrun_1', { + deploymentId: 'dpl_streamed', + }); + }); + it('resolves undefined when encryption is not configured', async () => { const getKey = memoizeEncryptionKey(worldWithKey(undefined), 'wrun_1'); await expect(getKey()).resolves.toBeUndefined(); diff --git a/packages/core/src/runtime/helpers.ts b/packages/core/src/runtime/helpers.ts index f24944478f..c82cdee697 100644 --- a/packages/core/src/runtime/helpers.ts +++ b/packages/core/src/runtime/helpers.ts @@ -8,6 +8,7 @@ import type { CreateEventRequest, Event, EventResult, + EventStreamObserver, HealthCheckPayload, ValidQueueName, WorkflowRun, @@ -594,7 +595,8 @@ function shouldRetryWithoutEventCursor( */ export async function loadWorkflowRunEvents( runId: string, - afterCursor?: string + afterCursor?: string, + onEvent?: EventStreamObserver ): Promise { const incremental = afterCursor !== undefined; return trace( @@ -630,6 +632,7 @@ export async function loadWorkflowRunEvents( sortOrder: 'asc', cursor: requestedCursor ?? undefined, }, + onEvent, }); } catch (error) { if ( @@ -1246,7 +1249,8 @@ export function getQueueOverhead(message: { requestedAt?: Date }) { */ export function memoizeEncryptionKey( world: World, - runOrId: WorkflowRun | string + runOrId: WorkflowRun | string, + context?: Record ): () => Promise { let cached: Promise | undefined; return () => { @@ -1257,7 +1261,7 @@ export function memoizeEncryptionKey( // here so TypeScript picks the right overload for each shape. const rawKey = typeof runOrId === 'string' - ? await world.getEncryptionKeyForRun?.(runOrId) + ? await world.getEncryptionKeyForRun?.(runOrId, context) : await world.getEncryptionKeyForRun?.(runOrId); // Resolve the *full* capability, not just the symmetric key: a run // reading its own event log may encounter sealed (`encp`) payloads diff --git a/packages/core/src/runtime/quickjs-partial-preload.test.ts b/packages/core/src/runtime/quickjs-partial-preload.test.ts index d9d3edf528..33f7b1df80 100644 --- a/packages/core/src/runtime/quickjs-partial-preload.test.ts +++ b/packages/core/src/runtime/quickjs-partial-preload.test.ts @@ -116,6 +116,7 @@ describe('QuickJS partial run_started preload', () => { expect(listEvents).toHaveBeenCalledWith({ runId, pagination: { sortOrder: 'asc', cursor: preloadCursor }, + onEvent: expect.any(Function), }); expect(runWorkflowWithQuickJS).toHaveBeenCalledWith( expect.objectContaining({ diff --git a/packages/core/src/step-delivery-ordering.test.ts b/packages/core/src/step-delivery-ordering.test.ts index 300cb6dba4..bbe81626a1 100644 --- a/packages/core/src/step-delivery-ordering.test.ts +++ b/packages/core/src/step-delivery-ordering.test.ts @@ -41,7 +41,7 @@ import { createSleep } from './workflow/sleep.js'; * - `wait_completed` resolves through a detached chain with a fixed, small * microtask-hop count (`workflow/sleep.ts`). A `step_completed` instead * resolves inside a serial `ctx.promiseQueue` slot that first hydrates the - * payload via `ReplayPayloadCache.getStepResult(...)`. That hop count is not + * payload via `ReplayPayloadCache.getPrimitiveValue(...)`. That hop count is not * fixed: the first hydration pays async decrypt/deserialize, while a later * replay sharing the same `ReplayPayloadCache` hits the * `primitiveStepResults` memo for small primitive results and resolves in diff --git a/packages/core/src/step.ts b/packages/core/src/step.ts index 1fcc4c11df..77e078613f 100644 --- a/packages/core/src/step.ts +++ b/packages/core/src/step.ts @@ -190,18 +190,25 @@ export function createUseStep(ctx: WorkflowOrchestratorContext) { ctx.pendingDeliveries++; ctx.promiseQueue = ctx.promiseQueue.then(async () => { try { - const prepared = await ctx.replayPayloadCache.prepareEventPayload( + rejection = await ctx.replayPayloadCache.getPrimitiveValue( event.eventId, 'error', - event.eventData.error - ); - rejection = await hydrateStepError( - event.eventData.error, - ctx.runId, - ctx.encryptionKey, - ctx.globalThis, - {}, - prepared + async () => { + const prepared = + await ctx.replayPayloadCache.prepareEventPayload( + event.eventId, + 'error', + event.eventData.error + ); + return hydrateStepError( + event.eventData.error, + ctx.runId, + ctx.encryptionKey, + ctx.globalThis, + {}, + prepared + ); + } ); } catch (hydrateErr) { // If hydration fails for any reason, fall back to a generic @@ -301,25 +308,27 @@ export function createUseStep(ctx: WorkflowOrchestratorContext) { ctx.pendingDeliveries++; ctx.promiseQueue = ctx.promiseQueue.then(async () => { try { - const hydratedResult = await ctx.replayPayloadCache.getStepResult( - completedEventId, - async () => { - const prepared = - await ctx.replayPayloadCache.prepareEventPayload( - completedEventId, - 'result', - serializedResult + const hydratedResult = + await ctx.replayPayloadCache.getPrimitiveValue( + completedEventId, + 'result', + async () => { + const prepared = + await ctx.replayPayloadCache.prepareEventPayload( + completedEventId, + 'result', + serializedResult + ); + return await hydrateStepReturnValue( + serializedResult, + ctx.runId, + ctx.encryptionKey, + ctx.globalThis, + {}, + prepared ); - return await hydrateStepReturnValue( - serializedResult, - ctx.runId, - ctx.encryptionKey, - ctx.globalThis, - {}, - prepared - ); - } - ); + } + ); outcome = { ok: true, value: hydratedResult as Result }; } catch (error) { outcome = { ok: false, error }; diff --git a/packages/core/src/workflow/hook.ts b/packages/core/src/workflow/hook.ts index bcb714d89d..aed3f44a02 100644 --- a/packages/core/src/workflow/hook.ts +++ b/packages/core/src/workflow/hook.ts @@ -352,19 +352,25 @@ export function createCreateHook(ctx: WorkflowOrchestratorContext) { | { ok: false; error: unknown }; ctx.promiseQueue = ctx.promiseQueue.then(async () => { try { - const prepared = - await ctx.replayPayloadCache.prepareEventPayload( - event.eventId, - 'payload', - event.eventData.payload - ); - const payload = await hydrateStepReturnValue( - event.eventData.payload, - ctx.runId, - ctx.encryptionKey, - ctx.globalThis, - {}, - prepared + const payload = await ctx.replayPayloadCache.getPrimitiveValue( + event.eventId, + 'payload', + async () => { + const prepared = + await ctx.replayPayloadCache.prepareEventPayload( + event.eventId, + 'payload', + event.eventData.payload + ); + return hydrateStepReturnValue( + event.eventData.payload, + ctx.runId, + ctx.encryptionKey, + ctx.globalThis, + {}, + prepared + ); + } ); hydrateOutcome = { ok: true, value: payload as T }; } catch (error) { @@ -425,18 +431,25 @@ export function createCreateHook(ctx: WorkflowOrchestratorContext) { ctx.pendingDeliveries++; ctx.promiseQueue = ctx.promiseQueue.then(async () => { try { - const prepared = await ctx.replayPayloadCache.prepareEventPayload( + const payload = await ctx.replayPayloadCache.getPrimitiveValue( event.eventId, 'payload', - event.eventData.payload - ); - const payload = await hydrateStepReturnValue( - event.eventData.payload, - ctx.runId, - ctx.encryptionKey, - ctx.globalThis, - {}, - prepared + async () => { + const prepared = + await ctx.replayPayloadCache.prepareEventPayload( + event.eventId, + 'payload', + event.eventData.payload + ); + return hydrateStepReturnValue( + event.eventData.payload, + ctx.runId, + ctx.encryptionKey, + ctx.globalThis, + {}, + prepared + ); + } ); outcome = { ok: true, value: payload as T }; } catch (error) { From ad13b2d6672f954ba2956872ef1b0cfabfbba053 Mon Sep 17 00:00:00 2001 From: Nathan Colosimo <110621881+NathanColosimo@users.noreply.github.com> Date: Thu, 13 Aug 2026 23:53:47 -0700 Subject: [PATCH 02/16] [core] Centralize replay event hydration --- packages/core/src/step.ts | 51 +++++++++++------------------- packages/core/src/workflow/hook.ts | 32 ++++++------------- 2 files changed, 29 insertions(+), 54 deletions(-) diff --git a/packages/core/src/step.ts b/packages/core/src/step.ts index 77e078613f..9e43d2740b 100644 --- a/packages/core/src/step.ts +++ b/packages/core/src/step.ts @@ -190,25 +190,19 @@ export function createUseStep(ctx: WorkflowOrchestratorContext) { ctx.pendingDeliveries++; ctx.promiseQueue = ctx.promiseQueue.then(async () => { try { - rejection = await ctx.replayPayloadCache.getPrimitiveValue( + rejection = await ctx.replayPayloadCache.getEventValue( event.eventId, 'error', - async () => { - const prepared = - await ctx.replayPayloadCache.prepareEventPayload( - event.eventId, - 'error', - event.eventData.error - ); - return hydrateStepError( + event.eventData.error, + (prepared) => + hydrateStepError( event.eventData.error, ctx.runId, ctx.encryptionKey, ctx.globalThis, {}, prepared - ); - } + ) ); } catch (hydrateErr) { // If hydration fails for any reason, fall back to a generic @@ -308,27 +302,20 @@ export function createUseStep(ctx: WorkflowOrchestratorContext) { ctx.pendingDeliveries++; ctx.promiseQueue = ctx.promiseQueue.then(async () => { try { - const hydratedResult = - await ctx.replayPayloadCache.getPrimitiveValue( - completedEventId, - 'result', - async () => { - const prepared = - await ctx.replayPayloadCache.prepareEventPayload( - completedEventId, - 'result', - serializedResult - ); - return await hydrateStepReturnValue( - serializedResult, - ctx.runId, - ctx.encryptionKey, - ctx.globalThis, - {}, - prepared - ); - } - ); + const hydratedResult = await ctx.replayPayloadCache.getEventValue( + completedEventId, + 'result', + serializedResult, + (prepared) => + hydrateStepReturnValue( + serializedResult, + ctx.runId, + ctx.encryptionKey, + ctx.globalThis, + {}, + prepared + ) + ); outcome = { ok: true, value: hydratedResult as Result }; } catch (error) { outcome = { ok: false, error }; diff --git a/packages/core/src/workflow/hook.ts b/packages/core/src/workflow/hook.ts index aed3f44a02..53d5e4ced2 100644 --- a/packages/core/src/workflow/hook.ts +++ b/packages/core/src/workflow/hook.ts @@ -352,25 +352,19 @@ export function createCreateHook(ctx: WorkflowOrchestratorContext) { | { ok: false; error: unknown }; ctx.promiseQueue = ctx.promiseQueue.then(async () => { try { - const payload = await ctx.replayPayloadCache.getPrimitiveValue( + const payload = await ctx.replayPayloadCache.getEventValue( event.eventId, 'payload', - async () => { - const prepared = - await ctx.replayPayloadCache.prepareEventPayload( - event.eventId, - 'payload', - event.eventData.payload - ); - return hydrateStepReturnValue( + event.eventData.payload, + (prepared) => + hydrateStepReturnValue( event.eventData.payload, ctx.runId, ctx.encryptionKey, ctx.globalThis, {}, prepared - ); - } + ) ); hydrateOutcome = { ok: true, value: payload as T }; } catch (error) { @@ -431,25 +425,19 @@ export function createCreateHook(ctx: WorkflowOrchestratorContext) { ctx.pendingDeliveries++; ctx.promiseQueue = ctx.promiseQueue.then(async () => { try { - const payload = await ctx.replayPayloadCache.getPrimitiveValue( + const payload = await ctx.replayPayloadCache.getEventValue( event.eventId, 'payload', - async () => { - const prepared = - await ctx.replayPayloadCache.prepareEventPayload( - event.eventId, - 'payload', - event.eventData.payload - ); - return hydrateStepReturnValue( + event.eventData.payload, + (prepared) => + hydrateStepReturnValue( event.eventData.payload, ctx.runId, ctx.encryptionKey, ctx.globalThis, {}, prepared - ); - } + ) ); outcome = { ok: true, value: payload as T }; } catch (error) { From 524c2d0972b0c8e301b68cfe9d81873de3e9ee6d Mon Sep 17 00:00:00 2001 From: Nathan Colosimo <110621881+NathanColosimo@users.noreply.github.com> Date: Fri, 14 Aug 2026 13:56:04 -0700 Subject: [PATCH 03/16] [core] Simplify replay payload caching --- packages/core/src/abort-consistency.test.ts | 2 +- packages/core/src/abort-controller.test.ts | 2 +- .../core/src/abort-replay-ordering.test.ts | 2 +- .../async-deserialization-ordering.test.ts | 2 +- .../src/delivery-barrier-coverage.test.ts | 2 +- .../core/src/replay-payload-cache.test.ts | 128 ++++++++++++------ packages/core/src/replay-payload-cache.ts | 97 +++++-------- packages/core/src/runtime.ts | 14 +- .../core/src/step-delivery-hop-count.test.ts | 8 +- .../core/src/step-delivery-ordering.test.ts | 10 +- .../src/step-hydration-memoization.test.ts | 8 +- packages/core/src/step.test.ts | 2 +- .../src/test-support/orchestrator-context.ts | 2 +- packages/core/src/workflow/hook.test.ts | 2 +- packages/core/src/workflow/sleep.test.ts | 2 +- 15 files changed, 147 insertions(+), 136 deletions(-) diff --git a/packages/core/src/abort-consistency.test.ts b/packages/core/src/abort-consistency.test.ts index ec93e93227..c1f2f5a4b9 100644 --- a/packages/core/src/abort-consistency.test.ts +++ b/packages/core/src/abort-consistency.test.ts @@ -37,7 +37,7 @@ function setupWorkflowContext(events: Event[]): WorkflowOrchestratorContext { return { runId: 'wrun_test', encryptionKey: undefined, - replayPayloadCache: new ReplayPayloadCache(undefined), + replayPayloadCache: new ReplayPayloadCache(), globalThis: context.globalThis, eventsConsumer: new EventsConsumer(events, { // Fake context: no deliveries are modeled, so the gate is a no-op here. diff --git a/packages/core/src/abort-controller.test.ts b/packages/core/src/abort-controller.test.ts index 3adf469366..96c4f8bc04 100644 --- a/packages/core/src/abort-controller.test.ts +++ b/packages/core/src/abort-controller.test.ts @@ -35,7 +35,7 @@ function setupWorkflowContext( return { runId: 'wrun_test', encryptionKey: undefined, - replayPayloadCache: new ReplayPayloadCache(undefined), + replayPayloadCache: new ReplayPayloadCache(), globalThis: context.globalThis, eventsConsumer: new EventsConsumer(events, { // Fake context: no deliveries are modeled, so the gate is a no-op here. diff --git a/packages/core/src/abort-replay-ordering.test.ts b/packages/core/src/abort-replay-ordering.test.ts index 8961a64925..06296ee362 100644 --- a/packages/core/src/abort-replay-ordering.test.ts +++ b/packages/core/src/abort-replay-ordering.test.ts @@ -70,7 +70,7 @@ function setupWorkflowContext(events: Event[]): WorkflowOrchestratorContext { return { runId: 'wrun_test', encryptionKey: undefined, - replayPayloadCache: new ReplayPayloadCache(undefined), + replayPayloadCache: new ReplayPayloadCache(), globalThis: context.globalThis, eventsConsumer: new EventsConsumer(events, { // Fake context: no deliveries are modeled, so the gate is a no-op here. diff --git a/packages/core/src/async-deserialization-ordering.test.ts b/packages/core/src/async-deserialization-ordering.test.ts index 41d4684563..b23346e3e7 100644 --- a/packages/core/src/async-deserialization-ordering.test.ts +++ b/packages/core/src/async-deserialization-ordering.test.ts @@ -50,7 +50,7 @@ function setupWorkflowContext(events: Event[]): WorkflowOrchestratorContext { return { runId: 'wrun_test', encryptionKey: undefined, - replayPayloadCache: new ReplayPayloadCache(undefined), + replayPayloadCache: new ReplayPayloadCache(), globalThis: context.globalThis, eventsConsumer: new EventsConsumer(events, { // Fake context: no deliveries are modeled, so the gate is a no-op here. diff --git a/packages/core/src/delivery-barrier-coverage.test.ts b/packages/core/src/delivery-barrier-coverage.test.ts index ee113b8ff0..27b29d2b5d 100644 --- a/packages/core/src/delivery-barrier-coverage.test.ts +++ b/packages/core/src/delivery-barrier-coverage.test.ts @@ -86,7 +86,7 @@ function setupWorkflowContext(events: Event[]): WorkflowOrchestratorContext { suspensionGeneration: 0, runId: 'wrun_test', encryptionKey: undefined, - replayPayloadCache: new ReplayPayloadCache(undefined), + replayPayloadCache: new ReplayPayloadCache(), globalThis: context.globalThis, eventsConsumer: new EventsConsumer(events, { // Fake context: no deliveries are modeled, so the gate is a no-op here. diff --git a/packages/core/src/replay-payload-cache.test.ts b/packages/core/src/replay-payload-cache.test.ts index dc7d77ddf8..e5c019671f 100644 --- a/packages/core/src/replay-payload-cache.test.ts +++ b/packages/core/src/replay-payload-cache.test.ts @@ -87,6 +87,18 @@ describe('ReplayPayloadCache', () => { expect(retained.data.buffer.byteLength).toBe(retained.data.byteLength); }); + it('hydrates and memoizes a primitive', async () => { + const payload = new Uint8Array([1]); + const hydrate = vi.fn(() => 42); + const cache = new ReplayPayloadCache(undefined, async (value) => value); + + await expect( + cache.getEventValue('evnt_one', 'result', payload, hydrate) + ).resolves.toBe(42); + expect(cache.getEventValue('evnt_one', 'result', payload, hydrate)).toBe(42); + expect(hydrate).toHaveBeenCalledOnce(); + }); + it('keeps a failed prewarm until its consumer observes it, then retries', async () => { const payload = new Uint8Array([1]); const run = makeRun(payload); @@ -96,10 +108,9 @@ describe('ReplayPayloadCache', () => { .mockResolvedValueOnce(payload); const cache = new ReplayPayloadCache(undefined, preparer); - await cache.prewarm(run, []); - await expect(cache.prepareWorkflowInput(run)).rejects.toThrow( - 'decrypt failed' - ); + cache.prewarm(run, []); + await Promise.resolve(); + expect(() => cache.prepareWorkflowInput(run)).toThrow('decrypt failed'); expect(preparer).toHaveBeenCalledOnce(); await expect(cache.prepareWorkflowInput(run)).resolves.toEqual({ @@ -121,30 +132,52 @@ describe('ReplayPayloadCache', () => { const run = makeRun(payloads[0]); const events = makeEvents(payloads.slice(1)); - const warming = cache.prewarm(run, events); + cache.prewarm(run, events); expect(preparer).toHaveBeenCalledTimes(4); for (const resolve of resolvers.reverse()) resolve(); - await warming; - - const allSettled = vi.spyOn(Promise, 'allSettled'); - await cache.prewarm(run, events); + await Promise.all([ + cache.prepareWorkflowInput(run), + ...events.map((event) => { + switch (event.eventType) { + case 'step_completed': + return cache.prepareEventPayload( + event.eventId, + 'result', + event.eventData?.result + ); + case 'step_failed': + return cache.prepareEventPayload( + event.eventId, + 'error', + event.eventData?.error + ); + case 'hook_received': + return cache.prepareEventPayload( + event.eventId, + 'payload', + event.eventData?.payload + ); + default: + throw new Error(`Unexpected event: ${event.eventType}`); + } + }), + ]); + + cache.prewarm(run, events); expect(preparer).toHaveBeenCalledTimes(4); - expect(allSettled).toHaveBeenLastCalledWith([]); - allSettled.mockRestore(); }); it('prepares streamed events synchronously inside the decoder callback', async () => { const payload = new Uint8Array([1]); const order: string[] = []; - const preparer = vi.fn((value) => { + const preparer = vi.fn((value) => { order.push('prepare'); - return { data: value }; + return value; }); const cache = new ReplayPayloadCache(undefined, preparer); const [event] = makeEvents([payload]); - const preparation = cache.observeEvent(event, () => order.push('start')); - expect(preparation).toBeDefined(); + cache.observeEvent(event, () => order.push('start')); expect(preparer).toHaveBeenCalledOnce(); expect(order).toEqual(['start', 'prepare']); @@ -152,25 +185,31 @@ describe('ReplayPayloadCache', () => { cache.observeEvent(event, () => order.push('cached-start')); expect(order).toEqual(['start', 'prepare']); - await expect(preparation).resolves.toEqual({ data: payload }); + expect(cache.prepareEventPayload(event.eventId, 'result', payload)).toEqual( + payload + ); }); it('prepares queued stream events as soon as the run key resolves', async () => { const payload = new Uint8Array([1]); - const preparer = vi.fn((value) => ({ data: value })); + const preparer = vi.fn((value) => value); let resolveKey!: (key: undefined) => void; const key = new Promise((resolve) => { resolveKey = resolve; }); - const cache = new ReplayPayloadCache(key, preparer); + const cache = ReplayPayloadCache.waitingForKey(key, preparer); const [event] = makeEvents([payload]); - const preparation = cache.observeEvent(event); - expect(preparation).toBeDefined(); + cache.observeEvent(event); expect(preparer).not.toHaveBeenCalled(); + const preparation = cache.prepareEventPayload( + event.eventId, + 'result', + payload + ); resolveKey(undefined); - await expect(preparation).resolves.toEqual({ data: payload }); + await expect(preparation).resolves.toEqual(payload); expect(preparer).toHaveBeenCalledOnce(); }); @@ -189,9 +228,7 @@ describe('ReplayPayloadCache', () => { const preparer = vi.fn(decodePayload); const cache = new ReplayPayloadCache(key, preparer); - const directPreparation = prepareReplayPayload(serialized, key); - expect(directPreparation).not.toBeInstanceOf(Promise); - await directPreparation; + await decodePayload(serialized, key); const prepared = await cache.prepareEventPayload( 'evnt_encrypted', @@ -264,74 +301,81 @@ describe('ReplayPayloadCache', () => { it('memoizes primitive step results, including undefined', async () => { for (const value of [0, false, '', null, undefined]) { - const cache = new ReplayPayloadCache(undefined); + const cache = new ReplayPayloadCache(); const hydrate = vi.fn().mockResolvedValue(value); expect( - await cache.getPrimitiveValue('evnt_result', 'result', hydrate) + await cache.getEventValue('evnt_result', 'result', undefined, hydrate) ).toBe(value); expect( - await cache.getPrimitiveValue('evnt_result', 'result', hydrate) + await cache.getEventValue('evnt_result', 'result', undefined, hydrate) ).toBe(value); expect(hydrate).toHaveBeenCalledOnce(); } }); it('isolates primitive values by event payload field', async () => { - const cache = new ReplayPayloadCache(undefined); + const cache = new ReplayPayloadCache(); const result = vi.fn().mockResolvedValue('result'); const error = vi.fn().mockResolvedValue('error'); await expect( - cache.getPrimitiveValue('evnt_shared', 'result', result) + cache.getEventValue('evnt_shared', 'result', undefined, result) ).resolves.toBe('result'); await expect( - cache.getPrimitiveValue('evnt_shared', 'error', error) + cache.getEventValue('evnt_shared', 'error', undefined, error) ).resolves.toBe('error'); - await expect( - cache.getPrimitiveValue('evnt_shared', 'result', result) - ).resolves.toBe('result'); + expect( + cache.getEventValue('evnt_shared', 'result', undefined, result) + ).toBe('result'); expect(result).toHaveBeenCalledOnce(); expect(error).toHaveBeenCalledOnce(); }); - it('rehydrates mutable and oversized step results', async () => { + it('rehydrates mutable results and memoizes primitives of any size', async () => { const oversized = 'x'.repeat(4097); for (const value of [{ count: 0 }, oversized]) { - const cache = new ReplayPayloadCache(undefined); + const cache = new ReplayPayloadCache(); const hydrate = vi .fn() .mockImplementation(async () => typeof value === 'object' ? { ...value } : value ); - const first = await cache.getPrimitiveValue( + const first = await cache.getEventValue( 'evnt_result', 'result', + undefined, hydrate ); - const second = await cache.getPrimitiveValue( + const second = await cache.getEventValue( 'evnt_result', 'result', + undefined, hydrate ); - expect(hydrate).toHaveBeenCalledTimes(2); - if (typeof value === 'object') expect(second).not.toBe(first); + if (typeof value === 'object') { + expect(hydrate).toHaveBeenCalledTimes(2); + expect(second).not.toBe(first); + } else { + expect(hydrate).toHaveBeenCalledOnce(); + expect(second).toBe(first); + } } }); it('does not memoize failed step hydration', async () => { - const cache = new ReplayPayloadCache(undefined); + const cache = new ReplayPayloadCache(); const hydrate = vi .fn() .mockRejectedValueOnce(new Error('boom')) .mockResolvedValueOnce('ok'); await expect( - cache.getPrimitiveValue('evnt_result', 'result', hydrate) + cache.getEventValue('evnt_result', 'result', undefined, hydrate) ).rejects.toThrow('boom'); await expect( - cache.getPrimitiveValue('evnt_result', 'result', hydrate) + cache.getEventValue('evnt_result', 'result', undefined, hydrate) ).resolves.toBe('ok'); expect(hydrate).toHaveBeenCalledTimes(2); }); diff --git a/packages/core/src/replay-payload-cache.ts b/packages/core/src/replay-payload-cache.ts index 5d3748c883..f235aa2738 100644 --- a/packages/core/src/replay-payload-cache.ts +++ b/packages/core/src/replay-payload-cache.ts @@ -42,8 +42,12 @@ type Preparation = | { state: 'pending'; promise: Promise } | { state: 'failed'; error: unknown }; -function isPrimitive(value: unknown): boolean { - return value === null || !['object', 'function'].includes(typeof value); +function isCacheablePrimitive(value: unknown): boolean { + const type = typeof value; + return ( + value === null || + (type !== 'object' && type !== 'function' && type !== 'symbol') + ); } /** @@ -55,46 +59,29 @@ function isPrimitive(value: unknown): boolean { * share and skip that repeated deserialization entirely. */ export class ReplayPayloadCache { + private key: KeyState; private readonly preparations = new Map(); - private readonly pendingPreparations = new Set< - Promise - >(); private readonly primitiveValues = new Map(); private nextUnscannedEventIndex = 0; - private constructor( - private key: KeyState, + constructor( + key?: DecryptionKey, private readonly preparer: ReplayPayloadPreparer = prepareReplayPayload ) { - if (key.state === 'pending') { - void key.promise.then( - (value) => this.resolveKey(value), - (error) => this.rejectKey(error) - ); - } - } - - static unencrypted( - preparer: ReplayPayloadPreparer = prepareReplayPayload - ): ReplayPayloadCache { - return new ReplayPayloadCache( - { state: 'ready', value: undefined }, - preparer - ); - } - - static withKey( - key: DecryptionKey, - preparer: ReplayPayloadPreparer = prepareReplayPayload - ): ReplayPayloadCache { - return new ReplayPayloadCache({ state: 'ready', value: key }, preparer); + this.key = { state: 'ready', value: key }; } static waitingForKey( key: Promise, preparer: ReplayPayloadPreparer = prepareReplayPayload ): ReplayPayloadCache { - return new ReplayPayloadCache({ state: 'pending', promise: key }, preparer); + const cache = new ReplayPayloadCache(undefined, preparer); + cache.key = { state: 'pending', promise: key }; + void key.then( + (value) => cache.resolveKey(value), + (error) => cache.rejectKey(error) + ); + return cache; } /** Start preparing an event as soon as its frame has been decoded. */ @@ -139,13 +126,10 @@ export class ReplayPayloadCache { /** * Start every preparation not already observed from the event stream. - * Returns a Promise only when a sealed or portable codec is still running. + * Consumers await the few codecs that cannot complete synchronously. */ - prewarm(workflowRun: WorkflowRun, events: Event[]): void | Promise { - this.start( - this.workflowInputKey(workflowRun.runId), - workflowRun.input - ); + prewarm(workflowRun: WorkflowRun, events: Event[]): void { + this.start(this.workflowInputKey(workflowRun.runId), workflowRun.input); for ( let index = this.nextUnscannedEventIndex; index < events.length; @@ -154,7 +138,6 @@ export class ReplayPayloadCache { this.observeEvent(events[index]); } this.nextUnscannedEventIndex = events.length; - return this.waitForPending(); } /** A corrected reload may insert events before the previous scan position. */ @@ -190,11 +173,7 @@ export class ReplayPayloadCache { return this.primitiveValues.get(cacheKey); } - const prepared = this.prepareEventPayload( - eventId, - field, - serializedValue - ); + const prepared = this.prepareEventPayload(eventId, field, serializedValue); const hydrateAndCache = (payload: PreparedReplayPayload) => { const hydrated = hydrate(payload); return hydrated instanceof Promise @@ -207,7 +186,7 @@ export class ReplayPayloadCache { } private cachePrimitive(cacheKey: string, value: unknown): unknown { - if (isPrimitive(value)) this.primitiveValues.set(cacheKey, value); + if (isCacheablePrimitive(value)) this.primitiveValues.set(cacheKey, value); return value; } @@ -250,10 +229,8 @@ export class ReplayPayloadCache { } this.preparations.set(cacheKey, { state: 'pending', promise: result }); - this.pendingPreparations.add(result); void result.then( (prepared) => { - this.pendingPreparations.delete(result); const current = this.preparations.get(cacheKey); if (current?.state === 'pending' && current.promise === result) { this.preparations.set(cacheKey, { @@ -263,7 +240,6 @@ export class ReplayPayloadCache { } }, (error) => { - this.pendingPreparations.delete(result); const current = this.preparations.get(cacheKey); if (current?.state === 'pending' && current.promise === result) { this.preparations.set(cacheKey, { state: 'failed', error }); @@ -284,14 +260,26 @@ export class ReplayPayloadCache { this.start(cacheKey, value); const preparation = this.preparations.get(cacheKey); if (!preparation) { - throw new Error(`Replay payload preparation was not started: ${cacheKey}`); + throw new Error( + `Replay payload preparation was not started: ${cacheKey}` + ); } switch (preparation.state) { case 'ready': return preparation.value; case 'pending': - return preparation.promise; + return preparation.promise.catch((error) => { + const current = this.preparations.get(cacheKey); + if ( + current?.state === 'failed' || + (current?.state === 'pending' && + current.promise === preparation.promise) + ) { + this.preparations.delete(cacheKey); + } + throw error; + }); case 'failed': this.preparations.delete(cacheKey); throw preparation.error; @@ -303,19 +291,6 @@ export class ReplayPayloadCache { } } - private waitForPending(): void | Promise { - if ( - this.key.state === 'pending' && - [...this.preparations.values()].some( - (preparation) => preparation.state === 'waiting' - ) - ) { - return this.key.promise.then(() => this.waitForPending()); - } - if (this.pendingPreparations.size === 0) return; - return Promise.allSettled([...this.pendingPreparations]).then(() => {}); - } - private resolveKey(value: DecryptionKey | undefined): void { if (this.key.state !== 'pending') return; this.key = { state: 'ready', value }; diff --git a/packages/core/src/runtime.ts b/packages/core/src/runtime.ts index e11965cf26..f088cd78f5 100644 --- a/packages/core/src/runtime.ts +++ b/packages/core/src/runtime.ts @@ -1063,9 +1063,8 @@ export function workflowEntrypoint( ({ runOrId, context }) => memoizeEncryptionKey(world, runOrId, context)() ); - const replayPayloadCache = new ReplayPayloadCache( - encryptionKeyPromise - ); + const replayPayloadCache = + ReplayPayloadCache.waitingForKey(encryptionKeyPromise); const observeReplayEvent = createReplayEventObserver({ runId, cache: replayPayloadCache, @@ -3018,14 +3017,7 @@ export function workflowEntrypoint( // appended-event consumption on resume; consumers still // deserialize and resolve in event order. const replayEvents = eventLog.events; - const payloadPrewarm = replayPayloadCache.prewarm( - workflowRun, - replayEvents - ); - // Consumers await their own prepared payloads in event - // order. Do not delay a suspension on speculative work - // for payloads this replay never touched. - void payloadPrewarm.catch(() => {}); + replayPayloadCache.prewarm(workflowRun, replayEvents); let workflowResult: WorkflowResumeResult = retainedSession ? await resumeWorkflow(retainedSession, eventLog.events) : { type: 'replay' }; diff --git a/packages/core/src/step-delivery-hop-count.test.ts b/packages/core/src/step-delivery-hop-count.test.ts index 7653f85d4e..65f628206f 100644 --- a/packages/core/src/step-delivery-hop-count.test.ts +++ b/packages/core/src/step-delivery-hop-count.test.ts @@ -44,7 +44,7 @@ import { createSleep } from './workflow/sleep.js'; function setupWorkflowContext( events: Event[], - replayPayloadCache: ReplayPayloadCache = new ReplayPayloadCache(undefined) + replayPayloadCache: ReplayPayloadCache = new ReplayPayloadCache() ): WorkflowOrchestratorContext { const context = createContext({ seed: 'test', @@ -229,7 +229,7 @@ describe('step delivery ordering is independent of consumer hop count: hook payl const spy = await slowHydration(); try { const events = await buildEventLog(); - const cache = new ReplayPayloadCache(undefined); + const cache = new ReplayPayloadCache(); const c1 = setupWorkflowContext(events, cache); const r1 = await runWithDiscontinuation(c1, body(c1, extraHops)); if (!WorkflowSuspension.is(r1.error)) { @@ -346,7 +346,7 @@ describe('step delivery ordering is independent of consumer hop count: wait comp const spy = await slowHydration(); try { const events = await buildWaitEventLog(); - const cache = new ReplayPayloadCache(undefined); + const cache = new ReplayPayloadCache(); const c1 = setupWorkflowContext(events, cache); const r1 = await runWithDiscontinuation(c1, waitBody(c1, extraHops)); if (!WorkflowSuspension.is(r1.error)) @@ -472,7 +472,7 @@ describe('step delivery ordering is independent of consumer hop count: step fail const spy = await slowHydration(); try { const events = await buildFailedEventLog(); - const cache = new ReplayPayloadCache(undefined); + const cache = new ReplayPayloadCache(); const c1 = setupWorkflowContext(events, cache); const r1 = await runWithDiscontinuation(c1, failedBody(c1, extraHops)); if (!WorkflowSuspension.is(r1.error)) { diff --git a/packages/core/src/step-delivery-ordering.test.ts b/packages/core/src/step-delivery-ordering.test.ts index bbe81626a1..18df1c7b77 100644 --- a/packages/core/src/step-delivery-ordering.test.ts +++ b/packages/core/src/step-delivery-ordering.test.ts @@ -41,7 +41,7 @@ import { createSleep } from './workflow/sleep.js'; * - `wait_completed` resolves through a detached chain with a fixed, small * microtask-hop count (`workflow/sleep.ts`). A `step_completed` instead * resolves inside a serial `ctx.promiseQueue` slot that first hydrates the - * payload via `ReplayPayloadCache.getPrimitiveValue(...)`. That hop count is not + * payload via `ReplayPayloadCache.getEventValue(...)`. That hop count is not * fixed: the first hydration pays async decrypt/deserialize, while a later * replay sharing the same `ReplayPayloadCache` hits the * `primitiveStepResults` memo for small primitive results and resolves in @@ -91,7 +91,7 @@ import { createSleep } from './workflow/sleep.js'; */ function setupWorkflowContext( events: Event[], - replayPayloadCache: ReplayPayloadCache = new ReplayPayloadCache(undefined) + replayPayloadCache: ReplayPayloadCache = new ReplayPayloadCache() ): WorkflowOrchestratorContext { const context = createContext({ seed: 'test', @@ -340,7 +340,7 @@ describe('step result delivery ordering across replays', () => { // One cache for both replays: production shares a single // `ReplayPayloadCache` across every replay of one queue delivery. - const sharedCache = new ReplayPayloadCache(undefined); + const sharedCache = new ReplayPayloadCache(); const firstCtx = setupWorkflowContext(events, sharedCache); const first = await runWithDiscontinuation( @@ -532,7 +532,7 @@ describe('step result delivery ordering across replays', () => { const hydration = delayHydration(); spy = await hydration.install(); const events = await buildEventLog(); - const sharedCache = new ReplayPayloadCache(undefined); + const sharedCache = new ReplayPayloadCache(); const firstCtx = setupWorkflowContext(events, sharedCache); const first = await runWithDiscontinuation( @@ -578,7 +578,7 @@ describe('step result delivery ordering across replays', () => { const hydration = delayHydration(); spy = await hydration.install(); const events = await buildEventLog(); - const sharedCache = new ReplayPayloadCache(undefined); + const sharedCache = new ReplayPayloadCache(); for (const replay of [1, 2]) { const ctx = setupWorkflowContext(events, sharedCache); diff --git a/packages/core/src/step-hydration-memoization.test.ts b/packages/core/src/step-hydration-memoization.test.ts index e8aabc1d0d..ede78c99b2 100644 --- a/packages/core/src/step-hydration-memoization.test.ts +++ b/packages/core/src/step-hydration-memoization.test.ts @@ -25,7 +25,7 @@ import { createContext } from './vm/index.js'; // the inline loop threads one cache across replay iterations. function setupWorkflowContext( events: Event[], - replayPayloadCache = new ReplayPayloadCache(undefined) + replayPayloadCache = new ReplayPayloadCache() ): WorkflowOrchestratorContext { const context = createContext({ seed: 'test', @@ -90,7 +90,7 @@ describe('step hydration memoization through the step consumer', () => { it('skips re-hydration of primitive step results on a second replay sharing the cache', async () => { const events = await makeStepEvents(); - const cache = new ReplayPayloadCache(undefined); + const cache = new ReplayPayloadCache(); const serialization = await import('./serialization.js'); const hydrateSpy = vi.spyOn(serialization, 'hydrateStepReturnValue'); @@ -122,7 +122,7 @@ describe('step hydration memoization through the step consumer', () => { it('preserves event-log resolution order on cache hits even with variable timing', async () => { const events = await makeStepEvents(); - const cache = new ReplayPayloadCache(undefined); + const cache = new ReplayPayloadCache(); // Replay 1: populate the cache (no timing games needed). const ctx1 = setupWorkflowContext(events, cache); @@ -168,7 +168,7 @@ describe('step hydration memoization through the step consumer', () => { createdAt: new Date(), }, ]; - const cache = new ReplayPayloadCache(undefined); + const cache = new ReplayPayloadCache(); // Replay 1: hydrate the object, then mutate it (as workflow code might). const ctx1 = setupWorkflowContext(events, cache); diff --git a/packages/core/src/step.test.ts b/packages/core/src/step.test.ts index 574e441110..cd759f9004 100644 --- a/packages/core/src/step.test.ts +++ b/packages/core/src/step.test.ts @@ -54,7 +54,7 @@ function setupWorkflowContext(events: Event[]): WorkflowOrchestratorContext { return { runId: 'wrun_test', encryptionKey: undefined, - replayPayloadCache: new ReplayPayloadCache(undefined), + replayPayloadCache: new ReplayPayloadCache(), globalThis: context.globalThis, eventsConsumer: new EventsConsumer(events, { // Fake context: no deliveries are modeled, so the gate is a no-op here. diff --git a/packages/core/src/test-support/orchestrator-context.ts b/packages/core/src/test-support/orchestrator-context.ts index 1ae7a78204..a9aa11f3e9 100644 --- a/packages/core/src/test-support/orchestrator-context.ts +++ b/packages/core/src/test-support/orchestrator-context.ts @@ -34,7 +34,7 @@ export function setupWorkflowContext( suspensionGeneration: 0, runId: 'wrun_test', encryptionKey: undefined, - replayPayloadCache: new ReplayPayloadCache(undefined), + replayPayloadCache: new ReplayPayloadCache(), globalThis: context.globalThis, eventsConsumer: new EventsConsumer(events, { // Fake context: no deliveries are modeled, so the gate is a no-op here. diff --git a/packages/core/src/workflow/hook.test.ts b/packages/core/src/workflow/hook.test.ts index c6c59727db..0ac3f008ae 100644 --- a/packages/core/src/workflow/hook.test.ts +++ b/packages/core/src/workflow/hook.test.ts @@ -39,7 +39,7 @@ function setupWorkflowContext(events: Event[]): WorkflowOrchestratorContext { runId: 'wrun_test', encryptionKey: undefined, worldCapabilities: { hookRetention: { active: true } }, - replayPayloadCache: new ReplayPayloadCache(undefined), + replayPayloadCache: new ReplayPayloadCache(), globalThis: context.globalThis, eventsConsumer: new EventsConsumer(events, { // Fake context: no deliveries are modeled, so the gate is a no-op here. diff --git a/packages/core/src/workflow/sleep.test.ts b/packages/core/src/workflow/sleep.test.ts index 7780bc05f1..139b185653 100644 --- a/packages/core/src/workflow/sleep.test.ts +++ b/packages/core/src/workflow/sleep.test.ts @@ -23,7 +23,7 @@ function setupWorkflowContext(events: Event[]): WorkflowOrchestratorContext { suspensionGeneration: 0, runId: 'wrun_test', encryptionKey: undefined, - replayPayloadCache: new ReplayPayloadCache(undefined), + replayPayloadCache: new ReplayPayloadCache(), globalThis: context.globalThis, // ctx.onWorkflowError is accessed via closure — it's defined below on the same object eventsConsumer: new EventsConsumer(events, { From 705271f9c7f60040bba2db2784676745f9a38a61 Mon Sep 17 00:00:00 2001 From: Nathan Colosimo <110621881+NathanColosimo@users.noreply.github.com> Date: Fri, 14 Aug 2026 14:05:41 -0700 Subject: [PATCH 04/16] [core] Name replay preparation states explicitly --- packages/core/src/replay-payload-cache.ts | 69 ++++++++++++----------- 1 file changed, 37 insertions(+), 32 deletions(-) diff --git a/packages/core/src/replay-payload-cache.ts b/packages/core/src/replay-payload-cache.ts index f235aa2738..60268c003a 100644 --- a/packages/core/src/replay-payload-cache.ts +++ b/packages/core/src/replay-payload-cache.ts @@ -32,14 +32,14 @@ async function prepareReplayPayload( type ReplayPayloadField = 'result' | 'error' | 'payload'; type KeyState = - | { state: 'pending'; promise: Promise } + | { state: 'loading'; promise: Promise } | { state: 'ready'; value: DecryptionKey | undefined } | { state: 'failed'; error: unknown }; type Preparation = - | { state: 'waiting'; value: Uint8Array } + | { state: 'waitingForKey'; value: Uint8Array } | { state: 'ready'; value: PreparedReplayPayload } - | { state: 'pending'; promise: Promise } + | { state: 'preparing'; promise: Promise } | { state: 'failed'; error: unknown }; function isCacheablePrimitive(value: unknown): boolean { @@ -76,7 +76,7 @@ export class ReplayPayloadCache { preparer: ReplayPayloadPreparer = prepareReplayPayload ): ReplayPayloadCache { const cache = new ReplayPayloadCache(undefined, preparer); - cache.key = { state: 'pending', promise: key }; + cache.key = { state: 'loading', promise: key }; void key.then( (value) => cache.resolveKey(value), (error) => cache.rejectKey(error) @@ -88,35 +88,35 @@ export class ReplayPayloadCache { observeEvent(event: Event, onPreparationStart?: () => void): void { switch (event.eventType) { case 'run_created': - this.start( + this.startPreparation( this.workflowInputKey(event.runId), event.eventData.input, onPreparationStart ); break; case 'run_started': - this.start( + this.startPreparation( this.workflowInputKey(event.runId), event.eventData?.input, onPreparationStart ); break; case 'step_completed': - this.start( + this.startPreparation( this.eventPayloadKey(event.eventId, 'result'), event.eventData?.result, onPreparationStart ); break; case 'step_failed': - this.start( + this.startPreparation( this.eventPayloadKey(event.eventId, 'error'), event.eventData?.error, onPreparationStart ); break; case 'hook_received': - this.start( + this.startPreparation( this.eventPayloadKey(event.eventId, 'payload'), event.eventData?.payload, onPreparationStart @@ -129,7 +129,10 @@ export class ReplayPayloadCache { * Consumers await the few codecs that cannot complete synchronously. */ prewarm(workflowRun: WorkflowRun, events: Event[]): void { - this.start(this.workflowInputKey(workflowRun.runId), workflowRun.input); + this.startPreparation( + this.workflowInputKey(workflowRun.runId), + workflowRun.input + ); for ( let index = this.nextUnscannedEventIndex; index < events.length; @@ -148,7 +151,7 @@ export class ReplayPayloadCache { prepareWorkflowInput( workflowRun: WorkflowRun ): PreparedReplayPayload | Promise { - return this.consume( + return this.getPreparedPayload( this.workflowInputKey(workflowRun.runId), workflowRun.input ); @@ -159,7 +162,7 @@ export class ReplayPayloadCache { field: ReplayPayloadField, value: unknown ): PreparedReplayPayload | Promise { - return this.consume(this.eventPayloadKey(eventId, field), value); + return this.getPreparedPayload(this.eventPayloadKey(eventId, field), value); } getEventValue( @@ -190,7 +193,7 @@ export class ReplayPayloadCache { return value; } - private start( + private startPreparation( cacheKey: string, value: unknown, onPreparationStart?: () => void @@ -201,11 +204,11 @@ export class ReplayPayloadCache { onPreparationStart?.(); switch (this.key.state) { - case 'pending': - this.preparations.set(cacheKey, { state: 'waiting', value }); + case 'loading': + this.preparations.set(cacheKey, { state: 'waitingForKey', value }); break; case 'ready': - this.prepare(cacheKey, value, this.key.value); + this.runPreparation(cacheKey, value, this.key.value); break; case 'failed': this.preparations.set(cacheKey, { @@ -216,7 +219,7 @@ export class ReplayPayloadCache { } } - private prepare( + private runPreparation( cacheKey: string, value: Uint8Array, key: DecryptionKey | undefined @@ -228,11 +231,11 @@ export class ReplayPayloadCache { return; } - this.preparations.set(cacheKey, { state: 'pending', promise: result }); + this.preparations.set(cacheKey, { state: 'preparing', promise: result }); void result.then( (prepared) => { const current = this.preparations.get(cacheKey); - if (current?.state === 'pending' && current.promise === result) { + if (current?.state === 'preparing' && current.promise === result) { this.preparations.set(cacheKey, { state: 'ready', value: prepared, @@ -241,7 +244,7 @@ export class ReplayPayloadCache { }, (error) => { const current = this.preparations.get(cacheKey); - if (current?.state === 'pending' && current.promise === result) { + if (current?.state === 'preparing' && current.promise === result) { this.preparations.set(cacheKey, { state: 'failed', error }); } } @@ -251,13 +254,13 @@ export class ReplayPayloadCache { } } - private consume( + private getPreparedPayload( cacheKey: string, value: unknown ): PreparedReplayPayload | Promise { if (!(value instanceof Uint8Array)) return { data: value }; - this.start(cacheKey, value); + this.startPreparation(cacheKey, value); const preparation = this.preparations.get(cacheKey); if (!preparation) { throw new Error( @@ -268,12 +271,12 @@ export class ReplayPayloadCache { switch (preparation.state) { case 'ready': return preparation.value; - case 'pending': + case 'preparing': return preparation.promise.catch((error) => { const current = this.preparations.get(cacheKey); if ( current?.state === 'failed' || - (current?.state === 'pending' && + (current?.state === 'preparing' && current.promise === preparation.promise) ) { this.preparations.delete(cacheKey); @@ -283,29 +286,31 @@ export class ReplayPayloadCache { case 'failed': this.preparations.delete(cacheKey); throw preparation.error; - case 'waiting': - if (this.key.state !== 'pending') { + case 'waitingForKey': + if (this.key.state !== 'loading') { throw new Error(`Replay payload key was not resolved: ${cacheKey}`); } - return this.key.promise.then(() => this.consume(cacheKey, value)); + return this.key.promise.then(() => + this.getPreparedPayload(cacheKey, value) + ); } } private resolveKey(value: DecryptionKey | undefined): void { - if (this.key.state !== 'pending') return; + if (this.key.state !== 'loading') return; this.key = { state: 'ready', value }; for (const [cacheKey, preparation] of this.preparations) { - if (preparation.state === 'waiting') { - this.prepare(cacheKey, preparation.value, value); + if (preparation.state === 'waitingForKey') { + this.runPreparation(cacheKey, preparation.value, value); } } } private rejectKey(error: unknown): void { - if (this.key.state !== 'pending') return; + if (this.key.state !== 'loading') return; this.key = { state: 'failed', error }; for (const [cacheKey, preparation] of this.preparations) { - if (preparation.state === 'waiting') { + if (preparation.state === 'waitingForKey') { this.preparations.set(cacheKey, { state: 'failed', error }); } } From 9643c7683a8eff650ee6f0a6346f33ebebaed809 Mon Sep 17 00:00:00 2001 From: Nathan Colosimo <110621881+NathanColosimo@users.noreply.github.com> Date: Fri, 14 Aug 2026 14:16:06 -0700 Subject: [PATCH 05/16] [core] Key replay payloads by event --- .../core/src/replay-payload-cache.test.ts | 57 +++++++------------ packages/core/src/replay-payload-cache.ts | 38 +++++-------- packages/core/src/step.ts | 2 - .../core/src/workflow/abort-controller.ts | 1 - packages/core/src/workflow/hook.ts | 2 - 5 files changed, 35 insertions(+), 65 deletions(-) diff --git a/packages/core/src/replay-payload-cache.test.ts b/packages/core/src/replay-payload-cache.test.ts index e5c019671f..1713e7dd8a 100644 --- a/packages/core/src/replay-payload-cache.test.ts +++ b/packages/core/src/replay-payload-cache.test.ts @@ -59,8 +59,8 @@ describe('ReplayPayloadCache', () => { const preparer = vi.fn(async (value) => value); const cache = new ReplayPayloadCache(undefined, preparer); - const first = cache.prepareEventPayload('evnt_one', 'result', payload); - const second = cache.prepareEventPayload('evnt_one', 'result', payload); + const first = cache.prepareEventPayload('evnt_one', payload); + const second = cache.prepareEventPayload('evnt_one', payload); expect(first).toBe(second); await expect(first).resolves.toEqual({ data: payload }); @@ -93,9 +93,9 @@ describe('ReplayPayloadCache', () => { const cache = new ReplayPayloadCache(undefined, async (value) => value); await expect( - cache.getEventValue('evnt_one', 'result', payload, hydrate) + cache.getEventValue('evnt_one', payload, hydrate) ).resolves.toBe(42); - expect(cache.getEventValue('evnt_one', 'result', payload, hydrate)).toBe(42); + expect(cache.getEventValue('evnt_one', payload, hydrate)).toBe(42); expect(hydrate).toHaveBeenCalledOnce(); }); @@ -142,19 +142,16 @@ describe('ReplayPayloadCache', () => { case 'step_completed': return cache.prepareEventPayload( event.eventId, - 'result', event.eventData?.result ); case 'step_failed': return cache.prepareEventPayload( event.eventId, - 'error', event.eventData?.error ); case 'hook_received': return cache.prepareEventPayload( event.eventId, - 'payload', event.eventData?.payload ); default: @@ -185,9 +182,7 @@ describe('ReplayPayloadCache', () => { cache.observeEvent(event, () => order.push('cached-start')); expect(order).toEqual(['start', 'prepare']); - expect(cache.prepareEventPayload(event.eventId, 'result', payload)).toEqual( - payload - ); + expect(cache.prepareEventPayload(event.eventId, payload)).toEqual(payload); }); it('prepares queued stream events as soon as the run key resolves', async () => { @@ -202,11 +197,7 @@ describe('ReplayPayloadCache', () => { cache.observeEvent(event); expect(preparer).not.toHaveBeenCalled(); - const preparation = cache.prepareEventPayload( - event.eventId, - 'result', - payload - ); + const preparation = cache.prepareEventPayload(event.eventId, payload); resolveKey(undefined); await expect(preparation).resolves.toEqual(payload); @@ -232,12 +223,10 @@ describe('ReplayPayloadCache', () => { const prepared = await cache.prepareEventPayload( 'evnt_encrypted', - 'result', serialized ); const samePrepared = await cache.prepareEventPayload( 'evnt_encrypted', - 'result', serialized ); const first = deserializePreparedReplayPayload(prepared) as { @@ -289,8 +278,8 @@ describe('ReplayPayloadCache', () => { const preparer = vi.fn(async (value) => value); const cache = new ReplayPayloadCache(undefined, preparer); - await cache.prepareEventPayload('evnt_legacy', 'result', legacy); - await cache.prepareEventPayload('evnt_legacy', 'result', legacy); + await cache.prepareEventPayload('evnt_legacy', legacy); + await cache.prepareEventPayload('evnt_legacy', legacy); expect(preparer).not.toHaveBeenCalled(); const events = makeEvents([legacy, legacy, legacy]); @@ -304,30 +293,30 @@ describe('ReplayPayloadCache', () => { const cache = new ReplayPayloadCache(); const hydrate = vi.fn().mockResolvedValue(value); - expect( - await cache.getEventValue('evnt_result', 'result', undefined, hydrate) - ).toBe(value); - expect( - await cache.getEventValue('evnt_result', 'result', undefined, hydrate) - ).toBe(value); + expect(await cache.getEventValue('evnt_result', undefined, hydrate)).toBe( + value + ); + expect(await cache.getEventValue('evnt_result', undefined, hydrate)).toBe( + value + ); expect(hydrate).toHaveBeenCalledOnce(); } }); - it('isolates primitive values by event payload field', async () => { + it('isolates primitive values by event id', async () => { const cache = new ReplayPayloadCache(); const result = vi.fn().mockResolvedValue('result'); const error = vi.fn().mockResolvedValue('error'); await expect( - cache.getEventValue('evnt_shared', 'result', undefined, result) + cache.getEventValue('evnt_result', undefined, result) ).resolves.toBe('result'); await expect( - cache.getEventValue('evnt_shared', 'error', undefined, error) + cache.getEventValue('evnt_error', undefined, error) ).resolves.toBe('error'); - expect( - cache.getEventValue('evnt_shared', 'result', undefined, result) - ).toBe('result'); + expect(cache.getEventValue('evnt_result', undefined, result)).toBe( + 'result' + ); expect(result).toHaveBeenCalledOnce(); expect(error).toHaveBeenCalledOnce(); }); @@ -344,13 +333,11 @@ describe('ReplayPayloadCache', () => { const first = await cache.getEventValue( 'evnt_result', - 'result', undefined, hydrate ); const second = await cache.getEventValue( 'evnt_result', - 'result', undefined, hydrate ); @@ -372,10 +359,10 @@ describe('ReplayPayloadCache', () => { .mockResolvedValueOnce('ok'); await expect( - cache.getEventValue('evnt_result', 'result', undefined, hydrate) + cache.getEventValue('evnt_result', undefined, hydrate) ).rejects.toThrow('boom'); await expect( - cache.getEventValue('evnt_result', 'result', undefined, hydrate) + cache.getEventValue('evnt_result', undefined, hydrate) ).resolves.toBe('ok'); expect(hydrate).toHaveBeenCalledTimes(2); }); diff --git a/packages/core/src/replay-payload-cache.ts b/packages/core/src/replay-payload-cache.ts index 60268c003a..0c2aecdf6c 100644 --- a/packages/core/src/replay-payload-cache.ts +++ b/packages/core/src/replay-payload-cache.ts @@ -29,7 +29,7 @@ async function prepareReplayPayload( return { data: compactOwnedBytes(prepared) }; } -type ReplayPayloadField = 'result' | 'error' | 'payload'; +const WORKFLOW_INPUT_CACHE_KEY = 'workflow-input'; type KeyState = | { state: 'loading'; promise: Promise } @@ -89,35 +89,35 @@ export class ReplayPayloadCache { switch (event.eventType) { case 'run_created': this.startPreparation( - this.workflowInputKey(event.runId), + WORKFLOW_INPUT_CACHE_KEY, event.eventData.input, onPreparationStart ); break; case 'run_started': this.startPreparation( - this.workflowInputKey(event.runId), + WORKFLOW_INPUT_CACHE_KEY, event.eventData?.input, onPreparationStart ); break; case 'step_completed': this.startPreparation( - this.eventPayloadKey(event.eventId, 'result'), + this.eventPayloadKey(event.eventId), event.eventData?.result, onPreparationStart ); break; case 'step_failed': this.startPreparation( - this.eventPayloadKey(event.eventId, 'error'), + this.eventPayloadKey(event.eventId), event.eventData?.error, onPreparationStart ); break; case 'hook_received': this.startPreparation( - this.eventPayloadKey(event.eventId, 'payload'), + this.eventPayloadKey(event.eventId), event.eventData?.payload, onPreparationStart ); @@ -129,10 +129,7 @@ export class ReplayPayloadCache { * Consumers await the few codecs that cannot complete synchronously. */ prewarm(workflowRun: WorkflowRun, events: Event[]): void { - this.startPreparation( - this.workflowInputKey(workflowRun.runId), - workflowRun.input - ); + this.startPreparation(WORKFLOW_INPUT_CACHE_KEY, workflowRun.input); for ( let index = this.nextUnscannedEventIndex; index < events.length; @@ -151,32 +148,27 @@ export class ReplayPayloadCache { prepareWorkflowInput( workflowRun: WorkflowRun ): PreparedReplayPayload | Promise { - return this.getPreparedPayload( - this.workflowInputKey(workflowRun.runId), - workflowRun.input - ); + return this.getPreparedPayload(WORKFLOW_INPUT_CACHE_KEY, workflowRun.input); } prepareEventPayload( eventId: string, - field: ReplayPayloadField, value: unknown ): PreparedReplayPayload | Promise { - return this.getPreparedPayload(this.eventPayloadKey(eventId, field), value); + return this.getPreparedPayload(this.eventPayloadKey(eventId), value); } getEventValue( eventId: string, - field: ReplayPayloadField, serializedValue: unknown, hydrate: (prepared: PreparedReplayPayload) => unknown | Promise ): unknown | Promise { - const cacheKey = this.eventPayloadKey(eventId, field); + const cacheKey = this.eventPayloadKey(eventId); if (this.primitiveValues.has(cacheKey)) { return this.primitiveValues.get(cacheKey); } - const prepared = this.prepareEventPayload(eventId, field, serializedValue); + const prepared = this.prepareEventPayload(eventId, serializedValue); const hydrateAndCache = (payload: PreparedReplayPayload) => { const hydrated = hydrate(payload); return hydrated instanceof Promise @@ -316,11 +308,7 @@ export class ReplayPayloadCache { } } - private workflowInputKey(runId: string): string { - return `run:${runId}:input`; - } - - private eventPayloadKey(eventId: string, field: ReplayPayloadField): string { - return `event:${eventId}:${field}`; + private eventPayloadKey(eventId: string): string { + return `event:${eventId}`; } } diff --git a/packages/core/src/step.ts b/packages/core/src/step.ts index 9e43d2740b..78a9379cee 100644 --- a/packages/core/src/step.ts +++ b/packages/core/src/step.ts @@ -192,7 +192,6 @@ export function createUseStep(ctx: WorkflowOrchestratorContext) { try { rejection = await ctx.replayPayloadCache.getEventValue( event.eventId, - 'error', event.eventData.error, (prepared) => hydrateStepError( @@ -304,7 +303,6 @@ export function createUseStep(ctx: WorkflowOrchestratorContext) { try { const hydratedResult = await ctx.replayPayloadCache.getEventValue( completedEventId, - 'result', serializedResult, (prepared) => hydrateStepReturnValue( diff --git a/packages/core/src/workflow/abort-controller.ts b/packages/core/src/workflow/abort-controller.ts index 4d5d9a08af..02b4f39bba 100644 --- a/packages/core/src/workflow/abort-controller.ts +++ b/packages/core/src/workflow/abort-controller.ts @@ -233,7 +233,6 @@ export function createCreateAbortController(ctx: WorkflowOrchestratorContext) { const prepared = await ctx.replayPayloadCache.prepareEventPayload( event.eventId, - 'payload', rawPayload ); const hydrated = (await hydrateStepReturnValue( diff --git a/packages/core/src/workflow/hook.ts b/packages/core/src/workflow/hook.ts index 53d5e4ced2..16e14ccedc 100644 --- a/packages/core/src/workflow/hook.ts +++ b/packages/core/src/workflow/hook.ts @@ -354,7 +354,6 @@ export function createCreateHook(ctx: WorkflowOrchestratorContext) { try { const payload = await ctx.replayPayloadCache.getEventValue( event.eventId, - 'payload', event.eventData.payload, (prepared) => hydrateStepReturnValue( @@ -427,7 +426,6 @@ export function createCreateHook(ctx: WorkflowOrchestratorContext) { try { const payload = await ctx.replayPayloadCache.getEventValue( event.eventId, - 'payload', event.eventData.payload, (prepared) => hydrateStepReturnValue( From a7f80caefdade6db6f0b0f2373d809da2b05985c Mon Sep 17 00:00:00 2001 From: Nathan Colosimo <110621881+NathanColosimo@users.noreply.github.com> Date: Fri, 14 Aug 2026 15:42:26 -0700 Subject: [PATCH 06/16] [core] Collapse replay cache state machinery --- packages/core/src/replay-payload-cache.ts | 140 ++++++---------------- 1 file changed, 39 insertions(+), 101 deletions(-) diff --git a/packages/core/src/replay-payload-cache.ts b/packages/core/src/replay-payload-cache.ts index 0c2aecdf6c..0265174491 100644 --- a/packages/core/src/replay-payload-cache.ts +++ b/packages/core/src/replay-payload-cache.ts @@ -10,7 +10,7 @@ import { recordCompression } from './serialization/telemetry.js'; type ReplayPayloadPreparer = ( data: Uint8Array, key: DecryptionKey | undefined -) => PreparedReplayPayload | Promise; +) => Uint8Array | Promise; /** Copy a view only when retaining it would also retain unrelated bytes. */ function compactOwnedBytes(data: Uint8Array): Uint8Array { @@ -22,26 +22,15 @@ function compactOwnedBytes(data: Uint8Array): Uint8Array { async function prepareReplayPayload( data: Uint8Array, key: DecryptionKey | undefined -): Promise { +): Promise { const compressionStats: CompressionStats = {}; const prepared = await decodePayload(data, key, compressionStats); await recordCompression(compressionStats, 'deserialize'); - return { data: compactOwnedBytes(prepared) }; + return prepared; } const WORKFLOW_INPUT_CACHE_KEY = 'workflow-input'; -type KeyState = - | { state: 'loading'; promise: Promise } - | { state: 'ready'; value: DecryptionKey | undefined } - | { state: 'failed'; error: unknown }; - -type Preparation = - | { state: 'waitingForKey'; value: Uint8Array } - | { state: 'ready'; value: PreparedReplayPayload } - | { state: 'preparing'; promise: Promise } - | { state: 'failed'; error: unknown }; - function isCacheablePrimitive(value: unknown): boolean { const type = typeof value; return ( @@ -59,27 +48,31 @@ function isCacheablePrimitive(value: unknown): boolean { * share and skip that repeated deserialization entirely. */ export class ReplayPayloadCache { - private key: KeyState; - private readonly preparations = new Map(); + private readonly preparations = new Map< + string, + Uint8Array | Promise | { readonly error: unknown } + >(); private readonly primitiveValues = new Map(); private nextUnscannedEventIndex = 0; + private encryptionKeyPromise?: Promise; constructor( - key?: DecryptionKey, + private encryptionKey?: DecryptionKey, private readonly preparer: ReplayPayloadPreparer = prepareReplayPayload - ) { - this.key = { state: 'ready', value: key }; - } + ) {} static waitingForKey( key: Promise, preparer: ReplayPayloadPreparer = prepareReplayPayload ): ReplayPayloadCache { const cache = new ReplayPayloadCache(undefined, preparer); - cache.key = { state: 'loading', promise: key }; + cache.encryptionKeyPromise = key; void key.then( - (value) => cache.resolveKey(value), - (error) => cache.rejectKey(error) + (value) => { + cache.encryptionKey = value; + cache.encryptionKeyPromise = undefined; + }, + () => {} ); return cache; } @@ -195,54 +188,33 @@ export class ReplayPayloadCache { } onPreparationStart?.(); - switch (this.key.state) { - case 'loading': - this.preparations.set(cacheKey, { state: 'waitingForKey', value }); - break; - case 'ready': - this.runPreparation(cacheKey, value, this.key.value); - break; - case 'failed': - this.preparations.set(cacheKey, { - state: 'failed', - error: this.key.error, - }); - break; - } + this.runPreparation(cacheKey, value); } - private runPreparation( - cacheKey: string, - value: Uint8Array, - key: DecryptionKey | undefined - ): void { + private runPreparation(cacheKey: string, value: Uint8Array): void { try { - const result = this.preparer(value, key); + const result = this.encryptionKeyPromise + ? this.encryptionKeyPromise.then((key) => this.preparer(value, key)) + : this.preparer(value, this.encryptionKey); if (!(result instanceof Promise)) { - this.preparations.set(cacheKey, { state: 'ready', value: result }); + this.preparations.set(cacheKey, compactOwnedBytes(result)); return; } - this.preparations.set(cacheKey, { state: 'preparing', promise: result }); - void result.then( + const compacted = result.then(compactOwnedBytes); + this.preparations.set(cacheKey, compacted); + void compacted.then( (prepared) => { const current = this.preparations.get(cacheKey); - if (current?.state === 'preparing' && current.promise === result) { - this.preparations.set(cacheKey, { - state: 'ready', - value: prepared, - }); - } + if (current === compacted) this.preparations.set(cacheKey, prepared); }, (error) => { const current = this.preparations.get(cacheKey); - if (current?.state === 'preparing' && current.promise === result) { - this.preparations.set(cacheKey, { state: 'failed', error }); - } + if (current === compacted) this.preparations.set(cacheKey, { error }); } ); } catch (error) { - this.preparations.set(cacheKey, { state: 'failed', error }); + this.preparations.set(cacheKey, { error }); } } @@ -253,59 +225,25 @@ export class ReplayPayloadCache { if (!(value instanceof Uint8Array)) return { data: value }; this.startPreparation(cacheKey, value); - const preparation = this.preparations.get(cacheKey); - if (!preparation) { + const prepared = this.preparations.get(cacheKey); + if (!prepared) { throw new Error( `Replay payload preparation was not started: ${cacheKey}` ); } - switch (preparation.state) { - case 'ready': - return preparation.value; - case 'preparing': - return preparation.promise.catch((error) => { - const current = this.preparations.get(cacheKey); - if ( - current?.state === 'failed' || - (current?.state === 'preparing' && - current.promise === preparation.promise) - ) { - this.preparations.delete(cacheKey); - } + if (prepared instanceof Uint8Array) return { data: prepared }; + if (prepared instanceof Promise) { + return prepared.then( + (data) => ({ data }), + (error) => { + this.preparations.delete(cacheKey); throw error; - }); - case 'failed': - this.preparations.delete(cacheKey); - throw preparation.error; - case 'waitingForKey': - if (this.key.state !== 'loading') { - throw new Error(`Replay payload key was not resolved: ${cacheKey}`); } - return this.key.promise.then(() => - this.getPreparedPayload(cacheKey, value) - ); - } - } - - private resolveKey(value: DecryptionKey | undefined): void { - if (this.key.state !== 'loading') return; - this.key = { state: 'ready', value }; - for (const [cacheKey, preparation] of this.preparations) { - if (preparation.state === 'waitingForKey') { - this.runPreparation(cacheKey, preparation.value, value); - } - } - } - - private rejectKey(error: unknown): void { - if (this.key.state !== 'loading') return; - this.key = { state: 'failed', error }; - for (const [cacheKey, preparation] of this.preparations) { - if (preparation.state === 'waitingForKey') { - this.preparations.set(cacheKey, { state: 'failed', error }); - } + ); } + this.preparations.delete(cacheKey); + throw prepared.error; } private eventPayloadKey(eventId: string): string { From a701f0a203f9b9263f8578aed9773826007a2aec Mon Sep 17 00:00:00 2001 From: Nathan Colosimo <110621881+NathanColosimo@users.noreply.github.com> Date: Fri, 14 Aug 2026 15:51:18 -0700 Subject: [PATCH 07/16] [core] Pass replay event loading options directly --- packages/core/src/runtime.ts | 47 +++++++++++------------ packages/core/src/runtime/helpers.test.ts | 28 +++++++++----- packages/core/src/runtime/helpers.ts | 17 ++++---- 3 files changed, 52 insertions(+), 40 deletions(-) diff --git a/packages/core/src/runtime.ts b/packages/core/src/runtime.ts index f088cd78f5..1c1067d61d 100644 --- a/packages/core/src/runtime.ts +++ b/packages/core/src/runtime.ts @@ -1697,7 +1697,7 @@ export function workflowEntrypoint( getStepFunction(incomingStepName)?.maxRetries ?? DEFAULT_STEP_MAX_RETRIES; if (metadata.attempt > bgMaxRetries + 1) { - const loaded = await loadWorkflowRunEvents(runId); + const loaded = await loadWorkflowRunEvents({ runId }); bgAuthoritativeAttempt = countStepStartedEvents( loaded.events, @@ -1812,7 +1812,7 @@ export function workflowEntrypoint( // Load events to check if all parallel steps are done. // Use cursor-based loading so the main loop can continue // incrementally from here. - const loaded = await loadWorkflowRunEvents(runId); + const loaded = await loadWorkflowRunEvents({ runId }); eventLog = nextEventLogLoad(loaded); // Check for pending steps: any step_created without @@ -2713,11 +2713,11 @@ export function workflowEntrypoint( if (eventLog.type === 'loadAfter') { appendEventLog( eventLog, - await loadWorkflowRunEvents( + await loadWorkflowRunEvents({ runId, - eventLog.cursor, - observeReplayEvent - ) + afterCursor: eventLog.cursor, + onEvent: observeReplayEvent, + }) ); eventLog = { ...eventLog, type: 'ready' }; } @@ -2770,13 +2770,14 @@ export function workflowEntrypoint( } if (eventLog.type !== 'ready') { - const page = await loadWorkflowRunEvents( + const page = await loadWorkflowRunEvents({ runId, - eventLog.type === 'loadAfter' - ? eventLog.cursor - : undefined, - observeReplayEvent - ); + afterCursor: + eventLog.type === 'loadAfter' + ? eventLog.cursor + : undefined, + onEvent: observeReplayEvent, + }); if (eventLog.type === 'loadAfter') { appendEventLog(eventLog, page); eventLog = { ...eventLog, type: 'ready' }; @@ -2893,11 +2894,11 @@ export function workflowEntrypoint( // not include the wait completion this handler just // attempted. if (eventLog.cursor) { - const page = await loadWorkflowRunEvents( + const page = await loadWorkflowRunEvents({ runId, - eventLog.cursor, - observeReplayEvent - ); + afterCursor: eventLog.cursor, + onEvent: observeReplayEvent, + }); const completedWaitIdsAfterCursor = new Set( page.events .filter((e) => e.eventType === 'wait_completed') @@ -2914,21 +2915,19 @@ export function workflowEntrypoint( appendEventLog(eventLog, page); } else { eventLog = { - ...(await loadWorkflowRunEvents( + ...(await loadWorkflowRunEvents({ runId, - undefined, - observeReplayEvent - )), + onEvent: observeReplayEvent, + })), type: 'ready', }; } } else { eventLog = { - ...(await loadWorkflowRunEvents( + ...(await loadWorkflowRunEvents({ runId, - undefined, - observeReplayEvent - )), + onEvent: observeReplayEvent, + })), type: 'ready', }; } diff --git a/packages/core/src/runtime/helpers.test.ts b/packages/core/src/runtime/helpers.test.ts index 0854f5d381..cc397963bd 100644 --- a/packages/core/src/runtime/helpers.test.ts +++ b/packages/core/src/runtime/helpers.test.ts @@ -414,7 +414,7 @@ describe('loadWorkflowRunEvents', () => { hasMore: false, }); - const result = await loadWorkflowRunEvents('wrun_test'); + const result = await loadWorkflowRunEvents({ runId: 'wrun_test' }); expect(result.events).toHaveLength(2); expect(result.cursor).toBe('eid:evnt_b'); @@ -447,7 +447,7 @@ describe('loadWorkflowRunEvents', () => { hasMore: false, }); - const result = await loadWorkflowRunEvents('wrun_test'); + const result = await loadWorkflowRunEvents({ runId: 'wrun_test' }); expect(result.events).toHaveLength(2); expect(result.cursor).toBe('eid:evnt_b'); @@ -461,7 +461,7 @@ describe('loadWorkflowRunEvents', () => { hasMore: false, }); - const result = await loadWorkflowRunEvents('wrun_test'); + const result = await loadWorkflowRunEvents({ runId: 'wrun_test' }); expect(result.events).toHaveLength(0); expect(result.cursor).toBeNull(); @@ -484,7 +484,7 @@ describe('loadWorkflowRunEvents', () => { hasMore: false, }); - const result = await loadWorkflowRunEvents('wrun_test'); + const result = await loadWorkflowRunEvents({ runId: 'wrun_test' }); expect(result.events.map((e) => e.eventId)).toEqual([ 'evnt_a', @@ -501,7 +501,10 @@ describe('loadWorkflowRunEvents', () => { hasMore: false, }); - const result = await loadWorkflowRunEvents('wrun_test', 'eid:evnt_z'); + const result = await loadWorkflowRunEvents({ + runId: 'wrun_test', + afterCursor: 'eid:evnt_z', + }); expect(result.events).toHaveLength(0); // Preserving the input cursor avoids the runtime treating "no new events @@ -521,7 +524,7 @@ describe('loadWorkflowRunEvents', () => { hasMore: false, }); - const result = await loadWorkflowRunEvents('wrun_test'); + const result = await loadWorkflowRunEvents({ runId: 'wrun_test' }); expect(result.events.map((event) => event.eventId)).toEqual([ 'evnt_a', @@ -540,7 +543,10 @@ describe('loadWorkflowRunEvents', () => { hasMore: false, }); - const result = await loadWorkflowRunEvents('wrun_test', 'opaque-cursor'); + const result = await loadWorkflowRunEvents({ + runId: 'wrun_test', + afterCursor: 'opaque-cursor', + }); expect(result.events.map((event) => event.eventId)).toEqual([ 'evnt_a', @@ -568,7 +574,9 @@ describe('loadWorkflowRunEvents', () => { hasMore: true, }); - await expect(loadWorkflowRunEvents('wrun_test')).rejects.toMatchObject({ + await expect( + loadWorkflowRunEvents({ runId: 'wrun_test' }) + ).rejects.toMatchObject({ code: 'WORLD_CONTRACT_ERROR', }); expect(eventsListMock).toHaveBeenCalledTimes(2); @@ -581,7 +589,9 @@ describe('loadWorkflowRunEvents', () => { hasMore: true, }); - await expect(loadWorkflowRunEvents('wrun_test')).rejects.toMatchObject({ + await expect( + loadWorkflowRunEvents({ runId: 'wrun_test' }) + ).rejects.toMatchObject({ code: 'WORLD_CONTRACT_ERROR', }); expect(eventsListMock).toHaveBeenCalledTimes(1); diff --git a/packages/core/src/runtime/helpers.ts b/packages/core/src/runtime/helpers.ts index c82cdee697..3a2fe82b1b 100644 --- a/packages/core/src/runtime/helpers.ts +++ b/packages/core/src/runtime/helpers.ts @@ -8,7 +8,6 @@ import type { CreateEventRequest, Event, EventResult, - EventStreamObserver, HealthCheckPayload, ValidQueueName, WorkflowRun, @@ -593,11 +592,15 @@ function shouldRetryWithoutEventCursor( * The returned cursor can be passed back in on a subsequent call for * incremental loading. */ -export async function loadWorkflowRunEvents( - runId: string, - afterCursor?: string, - onEvent?: EventStreamObserver -): Promise { +export async function loadWorkflowRunEvents({ + runId, + afterCursor, + onEvent, +}: { + runId: string; + afterCursor?: string; + onEvent?: (event: Event) => void; +}): Promise { const incremental = afterCursor !== undefined; return trace( incremental ? 'workflow.loadNewEvents' : 'workflow.loadEvents', @@ -984,7 +987,7 @@ export async function settleEventSlotGap( await new Promise((resolve) => setTimeout(resolve, SLOT_GAP_RECHECK_BASE_DELAY_MS * 2 ** attempt) ); - log = await loadWorkflowRunEvents(runId); + log = await loadWorkflowRunEvents({ runId }); gap = findEventSlotGap(log.events); } return { log, gap }; From 12f83478bdcf955156f0cefb733ba9361da576f2 Mon Sep 17 00:00:00 2001 From: Nathan Colosimo <110621881+NathanColosimo@users.noreply.github.com> Date: Fri, 14 Aug 2026 20:29:12 -0700 Subject: [PATCH 08/16] Simplify streamed replay payload preparation --- .../core/src/replay-payload-cache.test.ts | 151 ++++++------- packages/core/src/replay-payload-cache.ts | 200 ++++++------------ packages/core/src/runtime.ts | 122 +++++------ packages/core/src/runtime/helpers.ts | 33 +-- packages/core/src/workflow.ts | 2 +- .../core/src/workflow/abort-controller.ts | 22 +- 6 files changed, 226 insertions(+), 304 deletions(-) diff --git a/packages/core/src/replay-payload-cache.test.ts b/packages/core/src/replay-payload-cache.test.ts index 1713e7dd8a..22330b8053 100644 --- a/packages/core/src/replay-payload-cache.test.ts +++ b/packages/core/src/replay-payload-cache.test.ts @@ -1,7 +1,10 @@ import type { Event, WorkflowRun } from '@workflow/world'; import { assert, describe, expect, it, vi } from 'vitest'; import { importKey } from './encryption.js'; -import { ReplayPayloadCache } from './replay-payload-cache.js'; +import { + ReplayPayloadCache, + type ReplayPayloadPreparer, +} from './replay-payload-cache.js'; import { decodePayload } from './serialization/payload.js'; import { dehydrateStepReturnValue, @@ -56,34 +59,40 @@ function makeEvents(payloads: unknown[]): Event[] { describe('ReplayPayloadCache', () => { it('deduplicates preparation', async () => { const payload = new Uint8Array([1]); - const preparer = vi.fn(async (value) => value); + const preparer = vi.fn(async (value) => value); + const hydrate = vi.fn((prepared: unknown) => prepared); const cache = new ReplayPayloadCache(undefined, preparer); - const first = cache.prepareEventPayload('evnt_one', payload); - const second = cache.prepareEventPayload('evnt_one', payload); + const first = cache.getEventValue('evnt_one', payload, hydrate); + const second = cache.getEventValue('evnt_one', payload, hydrate); - expect(first).toBe(second); await expect(first).resolves.toEqual({ data: payload }); + await expect(second).resolves.toEqual({ data: payload }); expect(preparer).toHaveBeenCalledOnce(); + expect(hydrate).toHaveBeenCalledTimes(2); }); it('compacts prepared bytes before retaining them', async () => { - const backing = Buffer.alloc(64 * 1024); + const backing = new Uint8Array(64 * 1024); const prepared = backing.subarray(1024, 2048); const cache = new ReplayPayloadCache( undefined, - vi.fn(async () => prepared) + vi.fn(async () => prepared) ); - const retained = await cache.prepareEventPayload( + const retained = await cache.getEventValue( 'evnt_compact', - 'result', - new Uint8Array([1]) + new Uint8Array([1]), + (value) => value ); - assert(retained.data instanceof Uint8Array); - expect(Array.from(retained.data)).toEqual(Array.from(prepared)); - expect(retained.data).not.toBeInstanceOf(Buffer); + assert( + typeof retained === 'object' && + retained !== null && + 'data' in retained && + retained.data instanceof Uint8Array + ); + expect(retained.data).toEqual(prepared); expect(retained.data.buffer.byteLength).toBe(retained.data.byteLength); }); @@ -103,17 +112,17 @@ describe('ReplayPayloadCache', () => { const payload = new Uint8Array([1]); const run = makeRun(payload); const preparer = vi - .fn() + .fn() .mockRejectedValueOnce(new Error('decrypt failed')) .mockResolvedValueOnce(payload); const cache = new ReplayPayloadCache(undefined, preparer); - cache.prewarm(run, []); + cache.prepareAll(run, []); await Promise.resolve(); - expect(() => cache.prepareWorkflowInput(run)).toThrow('decrypt failed'); + await expect(cache.getWorkflowInput(run)).rejects.toThrow('decrypt failed'); expect(preparer).toHaveBeenCalledOnce(); - await expect(cache.prepareWorkflowInput(run)).resolves.toEqual({ + await expect(cache.getWorkflowInput(run)).resolves.toEqual({ data: payload, }); expect(preparer).toHaveBeenCalledTimes(2); @@ -122,7 +131,7 @@ describe('ReplayPayloadCache', () => { it('prewarms workflow, step, error, and hook payloads concurrently', async () => { const payloads = [0, 1, 2, 3].map((value) => new Uint8Array([value])); const resolvers: Array<() => void> = []; - const preparer = vi.fn( + const preparer = vi.fn( (value) => new Promise((resolve) => { resolvers.push(() => resolve(value)); @@ -132,27 +141,30 @@ describe('ReplayPayloadCache', () => { const run = makeRun(payloads[0]); const events = makeEvents(payloads.slice(1)); - cache.prewarm(run, events); + cache.prepareAll(run, events); expect(preparer).toHaveBeenCalledTimes(4); for (const resolve of resolvers.reverse()) resolve(); await Promise.all([ - cache.prepareWorkflowInput(run), + cache.getWorkflowInput(run), ...events.map((event) => { switch (event.eventType) { case 'step_completed': - return cache.prepareEventPayload( + return cache.getEventValue( event.eventId, - event.eventData?.result + event.eventData?.result, + (prepared) => prepared ); case 'step_failed': - return cache.prepareEventPayload( + return cache.getEventValue( event.eventId, - event.eventData?.error + event.eventData?.error, + (prepared) => prepared ); case 'hook_received': - return cache.prepareEventPayload( + return cache.getEventValue( event.eventId, - event.eventData?.payload + event.eventData?.payload, + (prepared) => prepared ); default: throw new Error(`Unexpected event: ${event.eventType}`); @@ -160,48 +172,30 @@ describe('ReplayPayloadCache', () => { }), ]); - cache.prewarm(run, events); + cache.prepareAll(run, events); expect(preparer).toHaveBeenCalledTimes(4); }); it('prepares streamed events synchronously inside the decoder callback', async () => { const payload = new Uint8Array([1]); const order: string[] = []; - const preparer = vi.fn((value) => { + const preparer = vi.fn((value) => { order.push('prepare'); return value; }); const cache = new ReplayPayloadCache(undefined, preparer); const [event] = makeEvents([payload]); - cache.observeEvent(event, () => order.push('start')); + cache.prepareEvent(event); expect(preparer).toHaveBeenCalledOnce(); - expect(order).toEqual(['start', 'prepare']); - - // Re-observing a cache hit does not move the preparation-span boundary. - cache.observeEvent(event, () => order.push('cached-start')); - expect(order).toEqual(['start', 'prepare']); - - expect(cache.prepareEventPayload(event.eventId, payload)).toEqual(payload); - }); - - it('prepares queued stream events as soon as the run key resolves', async () => { - const payload = new Uint8Array([1]); - const preparer = vi.fn((value) => value); - let resolveKey!: (key: undefined) => void; - const key = new Promise((resolve) => { - resolveKey = resolve; - }); - const cache = ReplayPayloadCache.waitingForKey(key, preparer); - const [event] = makeEvents([payload]); + expect(order).toEqual(['prepare']); - cache.observeEvent(event); - expect(preparer).not.toHaveBeenCalled(); - const preparation = cache.prepareEventPayload(event.eventId, payload); + cache.prepareEvent(event); + expect(order).toEqual(['prepare']); - resolveKey(undefined); - await expect(preparation).resolves.toEqual(payload); - expect(preparer).toHaveBeenCalledOnce(); + expect( + cache.getEventValue(event.eventId, payload, (prepared) => prepared) + ).toEqual({ data: payload }); }); it('caches real decrypt/decompress output but revives fresh objects', async () => { @@ -216,18 +210,20 @@ describe('ReplayPayloadCache', () => { false, true ); - const preparer = vi.fn(decodePayload); + const preparer = vi.fn(decodePayload); const cache = new ReplayPayloadCache(key, preparer); await decodePayload(serialized, key); - const prepared = await cache.prepareEventPayload( + const prepared = await cache.getEventValue( 'evnt_encrypted', - serialized + serialized, + (value) => value ); - const samePrepared = await cache.prepareEventPayload( + const samePrepared = await cache.getEventValue( 'evnt_encrypted', - serialized + serialized, + (value) => value ); const first = deserializePreparedReplayPayload(prepared) as { count: number; @@ -242,49 +238,36 @@ describe('ReplayPayloadCache', () => { expect(second.count).toBe(0); }); - it('rescans a log whose missing events were filled in below the scanned prefix', async () => { - // A stale-snapshot (412) restart replaces the log with a corrected one, so - // the events it was missing appear BELOW the length already scanned and - // shift every later position. Resuming from that length skips exactly the - // events the reload was for, which is what `resetScan` exists to prevent. + it('finds events inserted below a previously prepared prefix', () => { + // A stale-snapshot restart can replace the log with a corrected one whose + // missing events appear below the old tail. Full scans are cheap because + // event-id cache hits do no payload work. const payloads = [0, 1, 2].map((value) => new Uint8Array([value])); - const preparer = vi.fn(async (value) => value); + const preparer = vi.fn(async (value) => value); const cache = new ReplayPayloadCache(undefined, preparer); const run = makeRun(undefined); const [first, missing, second] = makeEvents(payloads); - await cache.prewarm(run, [first, second]); + cache.prepareAll(run, [first, second]); expect(preparer).toHaveBeenCalledTimes(2); - // Positional resume: `missing` sits inside the scanned prefix, so it is - // skipped and its payload is only prepared on demand. - await cache.prewarm(run, [first, missing, second]); - expect(preparer).toHaveBeenCalledTimes(2); - - cache.resetScan(); - await cache.prewarm(run, [first, missing, second]); - // Only the inserted event is new: the other two are keyed by event id and - // stay prepared across the rescan. + cache.prepareAll(run, [first, missing, second]); expect(preparer).toHaveBeenCalledTimes(3); - expect(preparer).toHaveBeenLastCalledWith( - payloads[1], - undefined, - expect.any(Object) - ); + expect(preparer).toHaveBeenLastCalledWith(payloads[1], undefined); }); - it('bypasses legacy values and ignores missing event data during prewarm', async () => { + it('bypasses legacy values and ignores missing event data during preparation', async () => { const legacy = [0, { value: 1 }]; - const preparer = vi.fn(async (value) => value); + const preparer = vi.fn(async (value) => value); const cache = new ReplayPayloadCache(undefined, preparer); - await cache.prepareEventPayload('evnt_legacy', legacy); - await cache.prepareEventPayload('evnt_legacy', legacy); + await cache.getEventValue('evnt_legacy', legacy, (prepared) => prepared); + await cache.getEventValue('evnt_legacy', legacy, (prepared) => prepared); expect(preparer).not.toHaveBeenCalled(); const events = makeEvents([legacy, legacy, legacy]); events[2] = { ...events[2], eventData: undefined } as unknown as Event; - await cache.prewarm(makeRun(legacy), events); + cache.prepareAll(makeRun(legacy), events); expect(preparer).not.toHaveBeenCalled(); }); diff --git a/packages/core/src/replay-payload-cache.ts b/packages/core/src/replay-payload-cache.ts index 0265174491..5866efd0ff 100644 --- a/packages/core/src/replay-payload-cache.ts +++ b/packages/core/src/replay-payload-cache.ts @@ -7,7 +7,7 @@ import { } from './serialization/payload.js'; import { recordCompression } from './serialization/telemetry.js'; -type ReplayPayloadPreparer = ( +export type ReplayPayloadPreparer = ( data: Uint8Array, key: DecryptionKey | undefined ) => Uint8Array | Promise; @@ -29,7 +29,9 @@ async function prepareReplayPayload( return prepared; } -const WORKFLOW_INPUT_CACHE_KEY = 'workflow-input'; +const WORKFLOW_INPUT = Symbol('workflow-input'); +type ReplayPayloadKey = string | typeof WORKFLOW_INPUT; +type CachedPreparation = Uint8Array | Promise; function isCacheablePrimitive(value: unknown): boolean { const type = typeof value; @@ -46,109 +48,54 @@ function isCacheablePrimitive(value: unknown): boolean { * Deserialization still runs against each VM's globals so object graphs and * Workflow objects remain realm-local. Primitive final values are safe to * share and skip that repeated deserialization entirely. + * + * Key lookup is deliberately outside this class. The runtime creates the + * cache once the run's key has resolved, then feeds it decoded events. Most + * Node preparation is synchronous; only codecs that are inherently async + * leave a Promise in the cache. */ export class ReplayPayloadCache { private readonly preparations = new Map< - string, - Uint8Array | Promise | { readonly error: unknown } + ReplayPayloadKey, + CachedPreparation >(); private readonly primitiveValues = new Map(); - private nextUnscannedEventIndex = 0; - private encryptionKeyPromise?: Promise; constructor( - private encryptionKey?: DecryptionKey, + private readonly encryptionKey?: DecryptionKey, private readonly preparer: ReplayPayloadPreparer = prepareReplayPayload ) {} - static waitingForKey( - key: Promise, - preparer: ReplayPayloadPreparer = prepareReplayPayload - ): ReplayPayloadCache { - const cache = new ReplayPayloadCache(undefined, preparer); - cache.encryptionKeyPromise = key; - void key.then( - (value) => { - cache.encryptionKey = value; - cache.encryptionKeyPromise = undefined; - }, - () => {} - ); - return cache; - } - - /** Start preparing an event as soon as its frame has been decoded. */ - observeEvent(event: Event, onPreparationStart?: () => void): void { + /** Prepare a payload as soon as its event frame has been decoded. */ + prepareEvent(event: Event): void { switch (event.eventType) { case 'run_created': - this.startPreparation( - WORKFLOW_INPUT_CACHE_KEY, - event.eventData.input, - onPreparationStart - ); + this.cachePayload(WORKFLOW_INPUT, event.eventData.input); break; case 'run_started': - this.startPreparation( - WORKFLOW_INPUT_CACHE_KEY, - event.eventData?.input, - onPreparationStart - ); + this.cachePayload(WORKFLOW_INPUT, event.eventData?.input); break; case 'step_completed': - this.startPreparation( - this.eventPayloadKey(event.eventId), - event.eventData?.result, - onPreparationStart - ); + this.cachePayload(event.eventId, event.eventData?.result); break; case 'step_failed': - this.startPreparation( - this.eventPayloadKey(event.eventId), - event.eventData?.error, - onPreparationStart - ); + this.cachePayload(event.eventId, event.eventData?.error); break; case 'hook_received': - this.startPreparation( - this.eventPayloadKey(event.eventId), - event.eventData?.payload, - onPreparationStart - ); - } - } - - /** - * Start every preparation not already observed from the event stream. - * Consumers await the few codecs that cannot complete synchronously. - */ - prewarm(workflowRun: WorkflowRun, events: Event[]): void { - this.startPreparation(WORKFLOW_INPUT_CACHE_KEY, workflowRun.input); - for ( - let index = this.nextUnscannedEventIndex; - index < events.length; - index++ - ) { - this.observeEvent(events[index]); + this.cachePayload(event.eventId, event.eventData?.payload); } - this.nextUnscannedEventIndex = events.length; } - /** A corrected reload may insert events before the previous scan position. */ - resetScan(): void { - this.nextUnscannedEventIndex = 0; + /** Prepare every payload not already seen through the event stream. */ + prepareAll(workflowRun: WorkflowRun, events: Event[]): void { + this.cachePayload(WORKFLOW_INPUT, workflowRun.input); + for (const event of events) this.prepareEvent(event); } - prepareWorkflowInput( + getWorkflowInput( workflowRun: WorkflowRun ): PreparedReplayPayload | Promise { - return this.getPreparedPayload(WORKFLOW_INPUT_CACHE_KEY, workflowRun.input); - } - - prepareEventPayload( - eventId: string, - value: unknown - ): PreparedReplayPayload | Promise { - return this.getPreparedPayload(this.eventPayloadKey(eventId), value); + return this.getPayload(WORKFLOW_INPUT, workflowRun.input); } getEventValue( @@ -156,97 +103,82 @@ export class ReplayPayloadCache { serializedValue: unknown, hydrate: (prepared: PreparedReplayPayload) => unknown | Promise ): unknown | Promise { - const cacheKey = this.eventPayloadKey(eventId); - if (this.primitiveValues.has(cacheKey)) { - return this.primitiveValues.get(cacheKey); + if (this.primitiveValues.has(eventId)) { + return this.primitiveValues.get(eventId); } - const prepared = this.prepareEventPayload(eventId, serializedValue); + const prepared = this.getPayload(eventId, serializedValue); const hydrateAndCache = (payload: PreparedReplayPayload) => { const hydrated = hydrate(payload); return hydrated instanceof Promise - ? hydrated.then((value) => this.cachePrimitive(cacheKey, value)) - : this.cachePrimitive(cacheKey, hydrated); + ? hydrated.then((value) => this.cachePrimitive(eventId, value)) + : this.cachePrimitive(eventId, hydrated); }; return prepared instanceof Promise ? prepared.then(hydrateAndCache) : hydrateAndCache(prepared); } - private cachePrimitive(cacheKey: string, value: unknown): unknown { - if (isCacheablePrimitive(value)) this.primitiveValues.set(cacheKey, value); + private cachePrimitive(eventId: string, value: unknown): unknown { + if (isCacheablePrimitive(value)) { + this.primitiveValues.set(eventId, value); + } return value; } - private startPreparation( - cacheKey: string, - value: unknown, - onPreparationStart?: () => void - ): void { + private cachePayload(cacheKey: ReplayPayloadKey, value: unknown): void { if (!(value instanceof Uint8Array) || this.preparations.has(cacheKey)) { return; } - onPreparationStart?.(); - this.runPreparation(cacheKey, value); - } - - private runPreparation(cacheKey: string, value: Uint8Array): void { + let preparation: CachedPreparation; try { - const result = this.encryptionKeyPromise - ? this.encryptionKeyPromise.then((key) => this.preparer(value, key)) - : this.preparer(value, this.encryptionKey); - if (!(result instanceof Promise)) { - this.preparations.set(cacheKey, compactOwnedBytes(result)); - return; - } + const prepared = this.preparer(value, this.encryptionKey); + preparation = + prepared instanceof Promise + ? prepared.then(compactOwnedBytes) + : compactOwnedBytes(prepared); + } catch (error) { + // Preparation is speculative. Preserve a synchronous failure for the + // ordered consumer without failing event loading or creating an + // unhandled rejection. + preparation = Promise.reject(error); + } + this.preparations.set(cacheKey, preparation); - const compacted = result.then(compactOwnedBytes); - this.preparations.set(cacheKey, compacted); - void compacted.then( + if (preparation instanceof Promise) { + void preparation.then( (prepared) => { - const current = this.preparations.get(cacheKey); - if (current === compacted) this.preparations.set(cacheKey, prepared); + if (this.preparations.get(cacheKey) === preparation) { + this.preparations.set(cacheKey, prepared); + } }, - (error) => { - const current = this.preparations.get(cacheKey); - if (current === compacted) this.preparations.set(cacheKey, { error }); - } + () => {} ); - } catch (error) { - this.preparations.set(cacheKey, { error }); } } - private getPreparedPayload( - cacheKey: string, + private getPayload( + cacheKey: ReplayPayloadKey, value: unknown ): PreparedReplayPayload | Promise { if (!(value instanceof Uint8Array)) return { data: value }; - this.startPreparation(cacheKey, value); + this.cachePayload(cacheKey, value); const prepared = this.preparations.get(cacheKey); if (!prepared) { - throw new Error( - `Replay payload preparation was not started: ${cacheKey}` - ); + throw new Error('Replay payload preparation was not cached'); } - if (prepared instanceof Uint8Array) return { data: prepared }; - if (prepared instanceof Promise) { - return prepared.then( - (data) => ({ data }), - (error) => { + if (!(prepared instanceof Promise)) return { data: prepared }; + return prepared.then( + (data) => ({ data }), + (error) => { + if (this.preparations.get(cacheKey) === prepared) { this.preparations.delete(cacheKey); - throw error; } - ); - } - this.preparations.delete(cacheKey); - throw prepared.error; - } - - private eventPayloadKey(eventId: string): string { - return `event:${eventId}`; + throw error; + } + ); } } diff --git a/packages/core/src/runtime.ts b/packages/core/src/runtime.ts index 1c1067d61d..9e3dee30cf 100644 --- a/packages/core/src/runtime.ts +++ b/packages/core/src/runtime.ts @@ -14,7 +14,7 @@ import { WorkflowRuntimeError, WorkflowWorldError, } from '@workflow/errors'; -import { once, setWorkflowBasePath, withResolvers } from '@workflow/utils'; +import { once, setWorkflowBasePath } from '@workflow/utils'; import { parseWorkflowName, workflowDisplayName, @@ -81,6 +81,7 @@ import { parseHealthCheckPayload, preconditionEventDelta, queueMessage, + resolveRunEncryptionKey, type SlotSnapshotParams, settleEventSlotGap, slotSnapshotParams, @@ -119,6 +120,7 @@ import { type WorldHandlers, } from './runtime/world.js'; import { dehydrateRunError } from './serialization.js'; +import type { DecryptionKey } from './serialization/encryption.js'; import { remapErrorStack } from './source-map.js'; import * as Attribute from './telemetry/semantic-conventions.js'; import { @@ -548,25 +550,6 @@ function replayEventDeploymentId(event: Event): string | undefined { return undefined; } -function createReplayEventObserver({ - runId, - cache, - resolveKey, -}: { - runId: string; - cache: ReplayPayloadCache; - resolveKey: ( - runOrId: WorkflowRun | string, - context?: Record - ) => void; -}): (event: Event) => void { - return (event) => { - const deploymentId = replayEventDeploymentId(event); - if (deploymentId) resolveKey(runId, { deploymentId }); - cache.observeEvent(event); - }; -} - /** * The whole retention predicate: keep the session only for a pure step * boundary (every suspension item is a step — any other item type, present @@ -1046,32 +1029,55 @@ export function workflowEntrypoint( // queue payload. This starts key resolution and payload // preparation before the remainder of the event log arrives, // without guessing the key for a cross-deployment run. - const { - promise: replayKeySource, - resolve: resolveReplayKeySource, - } = withResolvers<{ - runOrId: WorkflowRun | string; - context?: Record; - }>(); - const resolveReplayKey = ( + let replayEncryptionKey: + | Promise + | undefined; + let streamedPayloadCache: ReplayPayloadCache | undefined; + const eventsWaitingForKey: Event[] = []; + const activatePayloadCache = ( + key: DecryptionKey | undefined + ): ReplayPayloadCache => { + if (!streamedPayloadCache) { + streamedPayloadCache = new ReplayPayloadCache(key); + for (const event of eventsWaitingForKey) { + streamedPayloadCache.prepareEvent(event); + } + eventsWaitingForKey.length = 0; + } + return streamedPayloadCache; + }; + const getReplayEncryptionKey = ( runOrId: WorkflowRun | string, context?: Record - ): void => { - resolveReplayKeySource({ runOrId, context }); + ): Promise => { + if (!replayEncryptionKey) { + replayEncryptionKey = resolveRunEncryptionKey( + world, + runOrId, + context + ); + void replayEncryptionKey.then( + activatePayloadCache, + () => {} + ); + } + return replayEncryptionKey; + }; + const onReplayEvent = (event: Event): void => { + const deploymentId = replayEventDeploymentId(event); + if (deploymentId) { + void getReplayEncryptionKey(runId, { + deploymentId, + }); + } + if (streamedPayloadCache) { + streamedPayloadCache.prepareEvent(event); + } else { + eventsWaitingForKey.push(event); + } }; - const encryptionKeyPromise = replayKeySource.then( - ({ runOrId, context }) => - memoizeEncryptionKey(world, runOrId, context)() - ); - const replayPayloadCache = - ReplayPayloadCache.waitingForKey(encryptionKeyPromise); - const observeReplayEvent = createReplayEventObserver({ - runId, - cache: replayPayloadCache, - resolveKey: resolveReplayKey, - }); if (runInput?.deploymentId) { - resolveReplayKey(runId, { + void getReplayEncryptionKey(runId, { deploymentId: runInput.deploymentId, }); } @@ -1396,10 +1402,6 @@ export function workflowEntrypoint( // incremental load starts above the hole and never // returns it. eventLog = { type: 'loadAll' }; - // The corrected log inserts the missing events BELOW the - // length already scanned for payload prewarming, shifting - // every later position. Only a full rescan sees them. - replayPayloadCache.resetScan(); } runtimeLogger.warn( 'Event creation rejected as stale; restarting replay in-process', @@ -2047,7 +2049,7 @@ export function workflowEntrypoint( resumeId: hookResumeInput.resumeId, resumePayloadDigest: hookResumeInput.payloadDigest, preloadEvents: true, - onEvent: observeReplayEvent, + onEvent: onReplayEvent, } ); hookEnsured = true; @@ -2322,7 +2324,7 @@ export function workflowEntrypoint( }); const result = await createEvent(runStartedEvent, { requestId, - onEvent: observeReplayEvent, + onEvent: onReplayEvent, }); workflowRun = result.run; maxEventsLimit = clampMaxEvents(result.maxEvents); @@ -2607,7 +2609,7 @@ export function workflowEntrypoint( // do we fall back to reloading the complete log. if (eventLog.type !== 'loadAll' && ensuredEvent) { insertEventByEventId(eventLog.events, ensuredEvent); - observeReplayEvent(ensuredEvent); + onReplayEvent(ensuredEvent); } else { eventLog = { type: 'loadAll' }; } @@ -2617,8 +2619,10 @@ export function workflowEntrypoint( // Worlds that do not implement streamed observation still // resolve from the materialized run. This is also the final // cross-deployment-safe source of truth. - resolveReplayKey(workflowRun); - const encryptionKey = await encryptionKeyPromise; + const encryptionKey = + await getReplayEncryptionKey(workflowRun); + const replayPayloadCache = + activatePayloadCache(encryptionKey); // The live VM parked at the previous boundary, when the // retention decision kept it. null → this iteration cold- @@ -2716,7 +2720,7 @@ export function workflowEntrypoint( await loadWorkflowRunEvents({ runId, afterCursor: eventLog.cursor, - onEvent: observeReplayEvent, + onEvent: onReplayEvent, }) ); eventLog = { ...eventLog, type: 'ready' }; @@ -2776,7 +2780,7 @@ export function workflowEntrypoint( eventLog.type === 'loadAfter' ? eventLog.cursor : undefined, - onEvent: observeReplayEvent, + onEvent: onReplayEvent, }); if (eventLog.type === 'loadAfter') { appendEventLog(eventLog, page); @@ -2897,7 +2901,7 @@ export function workflowEntrypoint( const page = await loadWorkflowRunEvents({ runId, afterCursor: eventLog.cursor, - onEvent: observeReplayEvent, + onEvent: onReplayEvent, }); const completedWaitIdsAfterCursor = new Set( page.events @@ -2917,7 +2921,7 @@ export function workflowEntrypoint( eventLog = { ...(await loadWorkflowRunEvents({ runId, - onEvent: observeReplayEvent, + onEvent: onReplayEvent, })), type: 'ready', }; @@ -2926,7 +2930,7 @@ export function workflowEntrypoint( eventLog = { ...(await loadWorkflowRunEvents({ runId, - onEvent: observeReplayEvent, + onEvent: onReplayEvent, })), type: 'ready', }; @@ -3016,7 +3020,7 @@ export function workflowEntrypoint( // appended-event consumption on resume; consumers still // deserialize and resolve in event order. const replayEvents = eventLog.events; - replayPayloadCache.prewarm(workflowRun, replayEvents); + replayPayloadCache.prepareAll(workflowRun, replayEvents); let workflowResult: WorkflowResumeResult = retainedSession ? await resumeWorkflow(retainedSession, eventLog.events) : { type: 'replay' }; @@ -3308,12 +3312,10 @@ export function workflowEntrypoint( } if (suspensionResult.reportedEventCount > 0) { // Bump-and-report merged events BELOW the tail and - // re-sorted the array to slot order, shifting every - // position the prewarm scan had already recorded. + // re-sorted the array to slot order. // The cursor is deliberately left alone: the report // is a lower bound on what was skipped, so the next // incremental read still has to cover the same range. - replayPayloadCache.resetScan(); } // Open hooks/waits in the log as loaded for this diff --git a/packages/core/src/runtime/helpers.ts b/packages/core/src/runtime/helpers.ts index 3a2fe82b1b..f774ed0c6b 100644 --- a/packages/core/src/runtime/helpers.ts +++ b/packages/core/src/runtime/helpers.ts @@ -1250,6 +1250,24 @@ export function getQueueOverhead(message: { requestedAt?: Date }) { * outer try/catch to log and surface the issue; the queue's redelivery * semantics will retry the key fetch on the next attempt. */ +export async function resolveRunEncryptionKey( + world: World, + runOrId: WorkflowRun | string, + context?: Record +): Promise { + // The `getEncryptionKeyForRun` overload set takes either a `WorkflowRun` or + // a `runId: string` (with optional context). Branch here so TypeScript picks + // the right overload for each shape. + const rawKey = + typeof runOrId === 'string' + ? await world.getEncryptionKeyForRun?.(runOrId, context) + : await world.getEncryptionKeyForRun?.(runOrId); + // Resolve the *full* capability, not just the symmetric key: a run reading + // its own event log may encounter sealed (`encp`) payloads that another run + // wrote to it, and opening those needs the run's X25519 scalar as well. + return rawKey ? await deriveRunPayloadKeys(rawKey) : undefined; +} + export function memoizeEncryptionKey( world: World, runOrId: WorkflowRun | string, @@ -1258,20 +1276,7 @@ export function memoizeEncryptionKey( let cached: Promise | undefined; return () => { if (!cached) { - cached = (async () => { - // The `getEncryptionKeyForRun` overload set takes either a - // `WorkflowRun` or a `runId: string` (with optional context). Branch - // here so TypeScript picks the right overload for each shape. - const rawKey = - typeof runOrId === 'string' - ? await world.getEncryptionKeyForRun?.(runOrId, context) - : await world.getEncryptionKeyForRun?.(runOrId); - // Resolve the *full* capability, not just the symmetric key: a run - // reading its own event log may encounter sealed (`encp`) payloads - // that another run wrote to it, and opening those needs the run's - // X25519 scalar as well. - return rawKey ? await deriveRunPayloadKeys(rawKey) : undefined; - })(); + cached = resolveRunEncryptionKey(world, runOrId, context); } return cached; }; diff --git a/packages/core/src/workflow.ts b/packages/core/src/workflow.ts index 3bcd3f08fa..de9a248cf1 100644 --- a/packages/core/src/workflow.ts +++ b/packages/core/src/workflow.ts @@ -1085,7 +1085,7 @@ async function createWorkflowSession({ // workflow function subscribing its first step callbacks. let args: unknown[] = []; workflowContext.promiseQueue = workflowContext.promiseQueue.then(async () => { - const prepared = await replayPayloadCache.prepareWorkflowInput(workflowRun); + const prepared = await replayPayloadCache.getWorkflowInput(workflowRun); args = await hydrateWorkflowArguments( workflowRun.input, workflowRun.runId, diff --git a/packages/core/src/workflow/abort-controller.ts b/packages/core/src/workflow/abort-controller.ts index 02b4f39bba..6b7cacd6cb 100644 --- a/packages/core/src/workflow/abort-controller.ts +++ b/packages/core/src/workflow/abort-controller.ts @@ -230,18 +230,18 @@ export function createCreateAbortController(ctx: WorkflowOrchestratorContext) { try { if (rawPayload !== undefined) { try { - const prepared = - await ctx.replayPayloadCache.prepareEventPayload( - event.eventId, - rawPayload - ); - const hydrated = (await hydrateStepReturnValue( + const hydrated = (await ctx.replayPayloadCache.getEventValue( + event.eventId, rawPayload, - ctx.runId, - ctx.encryptionKey, - ctx.globalThis, - {}, - prepared + (prepared) => + hydrateStepReturnValue( + rawPayload, + ctx.runId, + ctx.encryptionKey, + ctx.globalThis, + {}, + prepared + ) )) as { reason?: unknown } | undefined; if ( hydrated && From 895dafd83a10fc6c4467f7b8641476d4630b513b Mon Sep 17 00:00:00 2001 From: Nathan Colosimo <110621881+NathanColosimo@users.noreply.github.com> Date: Fri, 14 Aug 2026 20:32:23 -0700 Subject: [PATCH 09/16] Sort replay runtime imports --- packages/core/src/runtime.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/core/src/runtime.ts b/packages/core/src/runtime.ts index 9e3dee30cf..1743bcf65b 100644 --- a/packages/core/src/runtime.ts +++ b/packages/core/src/runtime.ts @@ -119,8 +119,8 @@ import { getWorldHandlers, type WorldHandlers, } from './runtime/world.js'; -import { dehydrateRunError } from './serialization.js'; import type { DecryptionKey } from './serialization/encryption.js'; +import { dehydrateRunError } from './serialization.js'; import { remapErrorStack } from './source-map.js'; import * as Attribute from './telemetry/semantic-conventions.js'; import { From 079c0f79928d870cb8a9e2bb8b26a9ff45659831 Mon Sep 17 00:00:00 2001 From: Nathan Colosimo <110621881+NathanColosimo@users.noreply.github.com> Date: Tue, 18 Aug 2026 15:06:34 -0700 Subject: [PATCH 10/16] Fix replay payload cache bounds and scanning --- .../core/src/replay-payload-cache.test.ts | 53 +++++--- packages/core/src/replay-payload-cache.ts | 100 ++++++++------- packages/core/src/runtime.ts | 120 ++++++++++++------ packages/core/src/runtime/helpers.ts | 31 +---- 4 files changed, 173 insertions(+), 131 deletions(-) diff --git a/packages/core/src/replay-payload-cache.test.ts b/packages/core/src/replay-payload-cache.test.ts index 22330b8053..650f2351f5 100644 --- a/packages/core/src/replay-payload-cache.test.ts +++ b/packages/core/src/replay-payload-cache.test.ts @@ -176,10 +176,10 @@ describe('ReplayPayloadCache', () => { expect(preparer).toHaveBeenCalledTimes(4); }); - it('prepares streamed events synchronously inside the decoder callback', async () => { + it('starts streamed preparation inside the decoder callback', async () => { const payload = new Uint8Array([1]); const order: string[] = []; - const preparer = vi.fn((value) => { + const preparer = vi.fn(async (value) => { order.push('prepare'); return value; }); @@ -193,9 +193,9 @@ describe('ReplayPayloadCache', () => { cache.prepareEvent(event); expect(order).toEqual(['prepare']); - expect( + await expect( cache.getEventValue(event.eventId, payload, (prepared) => prepared) - ).toEqual({ data: payload }); + ).resolves.toEqual({ data: payload }); }); it('caches real decrypt/decompress output but revives fresh objects', async () => { @@ -213,7 +213,9 @@ describe('ReplayPayloadCache', () => { const preparer = vi.fn(decodePayload); const cache = new ReplayPayloadCache(key, preparer); - await decodePayload(serialized, key); + const directPreparation = decodePayload(serialized, key); + expect(directPreparation).toBeInstanceOf(Promise); + await directPreparation; const prepared = await cache.getEventValue( 'evnt_encrypted', @@ -238,10 +240,25 @@ describe('ReplayPayloadCache', () => { expect(second.count).toBe(0); }); - it('finds events inserted below a previously prepared prefix', () => { - // A stale-snapshot restart can replace the log with a corrected one whose - // missing events appear below the old tail. Full scans are cheap because - // event-id cache hits do no payload work. + it('prepares only events appended after the scanned prefix', () => { + const payloads = [0, 1, 2].map((value) => new Uint8Array([value])); + const preparer = vi.fn(async (value) => value); + const cache = new ReplayPayloadCache(undefined, preparer); + const run = makeRun(undefined); + const [first, second, third] = makeEvents(payloads); + const prepareEvent = vi.spyOn(cache, 'prepareEvent'); + + cache.prepareAll(run, [first, second]); + expect(preparer).toHaveBeenCalledTimes(2); + + prepareEvent.mockClear(); + cache.prepareAll(run, [first, second, third]); + expect(prepareEvent).toHaveBeenCalledOnce(); + expect(prepareEvent).toHaveBeenCalledWith(third); + expect(preparer).toHaveBeenCalledTimes(3); + }); + + it('rescans a corrected event log after reset', () => { const payloads = [0, 1, 2].map((value) => new Uint8Array([value])); const preparer = vi.fn(async (value) => value); const cache = new ReplayPayloadCache(undefined, preparer); @@ -249,8 +266,10 @@ describe('ReplayPayloadCache', () => { const [first, missing, second] = makeEvents(payloads); cache.prepareAll(run, [first, second]); + cache.prepareAll(run, [first, missing, second]); expect(preparer).toHaveBeenCalledTimes(2); + cache.resetScan(); cache.prepareAll(run, [first, missing, second]); expect(preparer).toHaveBeenCalledTimes(3); expect(preparer).toHaveBeenLastCalledWith(payloads[1], undefined); @@ -304,9 +323,14 @@ describe('ReplayPayloadCache', () => { expect(error).toHaveBeenCalledOnce(); }); - it('rehydrates mutable results and memoizes primitives of any size', async () => { - const oversized = 'x'.repeat(4097); - for (const value of [{ count: 0 }, oversized]) { + it('memoizes primitives within the budget and rehydrates larger results', async () => { + const commonText = 'x'.repeat(256 * 1024); + const oversizedText = 'x'.repeat(16 * 1024 * 1024 + 1); + for (const [value, expectedHydrations] of [ + [{ count: 0 }, 2], + [commonText, 1], + [oversizedText, 2], + ] as const) { const cache = new ReplayPayloadCache(); const hydrate = vi .fn() @@ -324,11 +348,10 @@ describe('ReplayPayloadCache', () => { undefined, hydrate ); + expect(hydrate).toHaveBeenCalledTimes(expectedHydrations); if (typeof value === 'object') { - expect(hydrate).toHaveBeenCalledTimes(2); expect(second).not.toBe(first); - } else { - expect(hydrate).toHaveBeenCalledOnce(); + } else if (expectedHydrations === 1) { expect(second).toBe(first); } } diff --git a/packages/core/src/replay-payload-cache.ts b/packages/core/src/replay-payload-cache.ts index 5866efd0ff..2719e5096f 100644 --- a/packages/core/src/replay-payload-cache.ts +++ b/packages/core/src/replay-payload-cache.ts @@ -10,14 +10,7 @@ import { recordCompression } from './serialization/telemetry.js'; export type ReplayPayloadPreparer = ( data: Uint8Array, key: DecryptionKey | undefined -) => Uint8Array | Promise; - -/** Copy a view only when retaining it would also retain unrelated bytes. */ -function compactOwnedBytes(data: Uint8Array): Uint8Array { - return data.byteOffset === 0 && data.byteLength === data.buffer.byteLength - ? data - : new Uint8Array(data); -} +) => Promise; async function prepareReplayPayload( data: Uint8Array, @@ -30,15 +23,29 @@ async function prepareReplayPayload( } const WORKFLOW_INPUT = Symbol('workflow-input'); +const MAX_MEMOIZED_PRIMITIVE_CHARACTERS = 16 * 1024 * 1024; type ReplayPayloadKey = string | typeof WORKFLOW_INPUT; -type CachedPreparation = Uint8Array | Promise; -function isCacheablePrimitive(value: unknown): boolean { +/** Copy a view only when retaining it would also retain unrelated bytes. */ +function compactOwnedBytes(data: Uint8Array): Uint8Array { + return data.byteOffset === 0 && data.byteLength === data.buffer.byteLength + ? data + : data.slice(); +} + +function cacheablePrimitiveCharacters(value: unknown): number | undefined { const type = typeof value; - return ( - value === null || - (type !== 'object' && type !== 'function' && type !== 'symbol') - ); + if (value === null) return 0; + if (type === 'string') { + return (value as string).length; + } + if (type === 'bigint') { + return (value as bigint).toString().length; + } + if (type === 'object' || type === 'function' || type === 'symbol') { + return undefined; + } + return 0; } /** @@ -47,19 +54,20 @@ function isCacheablePrimitive(value: unknown): boolean { * The cache retains VM-independent decrypt/decompress output across fresh VMs. * Deserialization still runs against each VM's globals so object graphs and * Workflow objects remain realm-local. Primitive final values are safe to - * share and skip that repeated deserialization entirely. + * share and skip that repeated deserialization entirely, within a bounded + * character budget because their prepared bytes remain cached too. * * Key lookup is deliberately outside this class. The runtime creates the - * cache once the run's key has resolved, then feeds it decoded events. Most - * Node preparation is synchronous; only codecs that are inherently async - * leave a Promise in the cache. + * cache once the run's key has resolved, then feeds it decoded events. */ export class ReplayPayloadCache { private readonly preparations = new Map< ReplayPayloadKey, - CachedPreparation + Promise >(); private readonly primitiveValues = new Map(); + private memoizedPrimitiveCharacters = 0; + private nextUnpreparedEventIndex = 0; constructor( private readonly encryptionKey?: DecryptionKey, @@ -89,7 +97,19 @@ export class ReplayPayloadCache { /** Prepare every payload not already seen through the event stream. */ prepareAll(workflowRun: WorkflowRun, events: Event[]): void { this.cachePayload(WORKFLOW_INPUT, workflowRun.input); - for (const event of events) this.prepareEvent(event); + for ( + let index = this.nextUnpreparedEventIndex; + index < events.length; + index++ + ) { + this.prepareEvent(events[index]); + } + this.nextUnpreparedEventIndex = events.length; + } + + /** Rescan after an event log is replaced or reordered. */ + resetScan(): void { + this.nextUnpreparedEventIndex = 0; } getWorkflowInput( @@ -120,9 +140,16 @@ export class ReplayPayloadCache { } private cachePrimitive(eventId: string, value: unknown): unknown { - if (isCacheablePrimitive(value)) { - this.primitiveValues.set(eventId, value); + const characters = cacheablePrimitiveCharacters(value); + if ( + characters === undefined || + this.memoizedPrimitiveCharacters + characters > + MAX_MEMOIZED_PRIMITIVE_CHARACTERS + ) { + return value; } + this.primitiveValues.set(eventId, value); + this.memoizedPrimitiveCharacters += characters; return value; } @@ -131,31 +158,11 @@ export class ReplayPayloadCache { return; } - let preparation: CachedPreparation; - try { - const prepared = this.preparer(value, this.encryptionKey); - preparation = - prepared instanceof Promise - ? prepared.then(compactOwnedBytes) - : compactOwnedBytes(prepared); - } catch (error) { - // Preparation is speculative. Preserve a synchronous failure for the - // ordered consumer without failing event loading or creating an - // unhandled rejection. - preparation = Promise.reject(error); - } + const preparation = this.preparer(value, this.encryptionKey).then( + compactOwnedBytes + ); this.preparations.set(cacheKey, preparation); - - if (preparation instanceof Promise) { - void preparation.then( - (prepared) => { - if (this.preparations.get(cacheKey) === preparation) { - this.preparations.set(cacheKey, prepared); - } - }, - () => {} - ); - } + void preparation.catch(() => {}); } private getPayload( @@ -170,7 +177,6 @@ export class ReplayPayloadCache { throw new Error('Replay payload preparation was not cached'); } - if (!(prepared instanceof Promise)) return { data: prepared }; return prepared.then( (data) => ({ data }), (error) => { diff --git a/packages/core/src/runtime.ts b/packages/core/src/runtime.ts index 1743bcf65b..1336a8ca3f 100644 --- a/packages/core/src/runtime.ts +++ b/packages/core/src/runtime.ts @@ -527,6 +527,24 @@ type ReplayEventLog = | ({ type: 'ready' } & LoadedEventLog) | ({ type: 'loadAfter'; cursor: string } & LoadedEventLog); +type ReplayEncryptionKeySource = + | { type: 'run'; run: WorkflowRun } + | { type: 'deployment'; deploymentId: string }; + +interface ReplayPayloads { + encryptionKey: DecryptionKey | undefined; + cache: ReplayPayloadCache; +} + +type ReplayPayloadCacheState = + | { type: 'waitingForKey'; events: Event[] } + | { + type: 'resolvingKey'; + events: Event[]; + payloads: Promise; + } + | { type: 'ready'; payloads: ReplayPayloads }; + function nextEventLogLoad(log: LoadedEventLog): ReplayEventLog { if (log.cursor === null) { return { type: 'loadAll' }; @@ -1029,55 +1047,72 @@ export function workflowEntrypoint( // queue payload. This starts key resolution and payload // preparation before the remainder of the event log arrives, // without guessing the key for a cross-deployment run. - let replayEncryptionKey: - | Promise - | undefined; - let streamedPayloadCache: ReplayPayloadCache | undefined; - const eventsWaitingForKey: Event[] = []; - const activatePayloadCache = ( - key: DecryptionKey | undefined - ): ReplayPayloadCache => { - if (!streamedPayloadCache) { - streamedPayloadCache = new ReplayPayloadCache(key); - for (const event of eventsWaitingForKey) { - streamedPayloadCache.prepareEvent(event); - } - eventsWaitingForKey.length = 0; - } - return streamedPayloadCache; + let replayPayloadState: ReplayPayloadCacheState = { + type: 'waitingForKey', + events: [], }; - const getReplayEncryptionKey = ( - runOrId: WorkflowRun | string, - context?: Record - ): Promise => { - if (!replayEncryptionKey) { - replayEncryptionKey = resolveRunEncryptionKey( - world, - runOrId, - context - ); - void replayEncryptionKey.then( - activatePayloadCache, - () => {} - ); + const getReplayPayloads = ( + source: ReplayEncryptionKeySource + ): ReplayPayloads | Promise => { + switch (replayPayloadState.type) { + case 'waitingForKey': { + const events = replayPayloadState.events; + const encryptionKey = + source.type === 'run' + ? resolveRunEncryptionKey(world, source.run) + : resolveRunEncryptionKey(world, runId, { + deploymentId: source.deploymentId, + }); + const resolving = encryptionKey.then( + (encryptionKey) => { + const cache = new ReplayPayloadCache(encryptionKey); + for (const event of events) { + cache.prepareEvent(event); + } + const payloads = { encryptionKey, cache }; + replayPayloadState = { + type: 'ready', + payloads, + }; + return payloads; + } + ); + replayPayloadState = { + type: 'resolvingKey', + events, + payloads: resolving, + }; + void resolving.catch(() => {}); + return resolving; + } + case 'resolvingKey': + case 'ready': + return replayPayloadState.payloads; } - return replayEncryptionKey; + replayPayloadState satisfies never; }; const onReplayEvent = (event: Event): void => { const deploymentId = replayEventDeploymentId(event); if (deploymentId) { - void getReplayEncryptionKey(runId, { + void getReplayPayloads({ + type: 'deployment', deploymentId, }); } - if (streamedPayloadCache) { - streamedPayloadCache.prepareEvent(event); - } else { - eventsWaitingForKey.push(event); + switch (replayPayloadState.type) { + case 'waitingForKey': + case 'resolvingKey': + replayPayloadState.events.push(event); + return; + case 'ready': + replayPayloadState.payloads.cache.prepareEvent(event); + return; } + replayPayloadState satisfies never; }; if (runInput?.deploymentId) { - void getReplayEncryptionKey(runId, { + void getReplayPayloads({ + type: 'deployment', deploymentId: runInput.deploymentId, }); } @@ -1402,6 +1437,7 @@ export function workflowEntrypoint( // incremental load starts above the hole and never // returns it. eventLog = { type: 'loadAll' }; + replayPayloadCache.resetScan(); } runtimeLogger.warn( 'Event creation rejected as stale; restarting replay in-process', @@ -2619,10 +2655,11 @@ export function workflowEntrypoint( // Worlds that do not implement streamed observation still // resolve from the materialized run. This is also the final // cross-deployment-safe source of truth. - const encryptionKey = - await getReplayEncryptionKey(workflowRun); - const replayPayloadCache = - activatePayloadCache(encryptionKey); + const { encryptionKey, cache: replayPayloadCache } = + await getReplayPayloads({ + type: 'run', + run: workflowRun, + }); // The live VM parked at the previous boundary, when the // retention decision kept it. null → this iteration cold- @@ -3316,6 +3353,7 @@ export function workflowEntrypoint( // The cursor is deliberately left alone: the report // is a lower bound on what was skipped, so the next // incremental read still has to cover the same range. + replayPayloadCache.resetScan(); } // Open hooks/waits in the log as loaded for this diff --git a/packages/core/src/runtime/helpers.ts b/packages/core/src/runtime/helpers.ts index f774ed0c6b..939369b329 100644 --- a/packages/core/src/runtime/helpers.ts +++ b/packages/core/src/runtime/helpers.ts @@ -1221,34 +1221,8 @@ export function getQueueOverhead(message: { requestedAt?: Date }) { } /** - * Returns a memoized accessor for a run's full encryption capability. - * - * The first call resolves the run's key material via - * `world.getEncryptionKeyForRun` (which may do HKDF derivation locally on - * Vercel, or a network fetch from external contexts) and derives a - * {@link PayloadKey} from it; subsequent calls await the same cached promise. - * If the world doesn't support encryption or the run has no key configured, - * the cached value is `undefined`. - * - * The resolved value is deliberately the *full* capability — the symmetric AES - * key plus the run's X25519 keypair — not just a `CryptoKey`. A run reading - * its own event log can encounter sealed (`encp`) payloads that another run - * wrote to it (a cross-deployment hook resumption, say), and opening those - * needs the keypair. Resolving only the symmetric key would leave those - * payloads unopenable and wedge the run. - * - * Used by step / workflow handlers to defer the (potentially expensive) - * key fetch until the first code path that actually needs it — typically - * input hydration on the success path, or error dehydration on a failure - * path. Both paths can race-call the accessor without triggering duplicate - * fetches. - * - * Errors thrown by `getEncryptionKeyForRun` propagate to every caller - * (the cached promise rejects). This is intentional: when encryption is - * configured, we never want to silently fall back to plaintext - * serialization. A propagated error in an event-emission path leaves the - * outer try/catch to log and surface the issue; the queue's redelivery - * semantics will retry the key fetch on the next attempt. + * Resolves a run's full encryption capability: its symmetric key and X25519 + * keypair. The keypair is required to open sealed cross-deployment payloads. */ export async function resolveRunEncryptionKey( world: World, @@ -1268,6 +1242,7 @@ export async function resolveRunEncryptionKey( return rawKey ? await deriveRunPayloadKeys(rawKey) : undefined; } +/** Returns a lazy accessor that shares the first key lookup and its outcome. */ export function memoizeEncryptionKey( world: World, runOrId: WorkflowRun | string, From 90f9413eb50000e674ed1660b54bbe9d7b46fb41 Mon Sep 17 00:00:00 2001 From: Nathan Colosimo <110621881+NathanColosimo@users.noreply.github.com> Date: Thu, 20 Aug 2026 10:26:57 -0700 Subject: [PATCH 11/16] perf(core): reuse primitives without a size cap --- .../core/src/replay-payload-cache.test.ts | 4 +-- packages/core/src/replay-payload-cache.ts | 32 ++++--------------- 2 files changed, 9 insertions(+), 27 deletions(-) diff --git a/packages/core/src/replay-payload-cache.test.ts b/packages/core/src/replay-payload-cache.test.ts index 650f2351f5..b2f192aad3 100644 --- a/packages/core/src/replay-payload-cache.test.ts +++ b/packages/core/src/replay-payload-cache.test.ts @@ -323,13 +323,13 @@ describe('ReplayPayloadCache', () => { expect(error).toHaveBeenCalledOnce(); }); - it('memoizes primitives within the budget and rehydrates larger results', async () => { + it('memoizes primitives of any size and rehydrates object results', async () => { const commonText = 'x'.repeat(256 * 1024); const oversizedText = 'x'.repeat(16 * 1024 * 1024 + 1); for (const [value, expectedHydrations] of [ [{ count: 0 }, 2], [commonText, 1], - [oversizedText, 2], + [oversizedText, 1], ] as const) { const cache = new ReplayPayloadCache(); const hydrate = vi diff --git a/packages/core/src/replay-payload-cache.ts b/packages/core/src/replay-payload-cache.ts index 2719e5096f..dd40511ec1 100644 --- a/packages/core/src/replay-payload-cache.ts +++ b/packages/core/src/replay-payload-cache.ts @@ -23,7 +23,6 @@ async function prepareReplayPayload( } const WORKFLOW_INPUT = Symbol('workflow-input'); -const MAX_MEMOIZED_PRIMITIVE_CHARACTERS = 16 * 1024 * 1024; type ReplayPayloadKey = string | typeof WORKFLOW_INPUT; /** Copy a view only when retaining it would also retain unrelated bytes. */ @@ -33,19 +32,12 @@ function compactOwnedBytes(data: Uint8Array): Uint8Array { : data.slice(); } -function cacheablePrimitiveCharacters(value: unknown): number | undefined { +function isCacheablePrimitive(value: unknown): boolean { const type = typeof value; - if (value === null) return 0; - if (type === 'string') { - return (value as string).length; - } - if (type === 'bigint') { - return (value as bigint).toString().length; - } - if (type === 'object' || type === 'function' || type === 'symbol') { - return undefined; - } - return 0; + return ( + value === null || + (type !== 'object' && type !== 'function' && type !== 'symbol') + ); } /** @@ -54,8 +46,7 @@ function cacheablePrimitiveCharacters(value: unknown): number | undefined { * The cache retains VM-independent decrypt/decompress output across fresh VMs. * Deserialization still runs against each VM's globals so object graphs and * Workflow objects remain realm-local. Primitive final values are safe to - * share and skip that repeated deserialization entirely, within a bounded - * character budget because their prepared bytes remain cached too. + * share and skip that repeated deserialization entirely. * * Key lookup is deliberately outside this class. The runtime creates the * cache once the run's key has resolved, then feeds it decoded events. @@ -66,7 +57,6 @@ export class ReplayPayloadCache { Promise >(); private readonly primitiveValues = new Map(); - private memoizedPrimitiveCharacters = 0; private nextUnpreparedEventIndex = 0; constructor( @@ -140,16 +130,8 @@ export class ReplayPayloadCache { } private cachePrimitive(eventId: string, value: unknown): unknown { - const characters = cacheablePrimitiveCharacters(value); - if ( - characters === undefined || - this.memoizedPrimitiveCharacters + characters > - MAX_MEMOIZED_PRIMITIVE_CHARACTERS - ) { - return value; - } + if (!isCacheablePrimitive(value)) return value; this.primitiveValues.set(eventId, value); - this.memoizedPrimitiveCharacters += characters; return value; } From dbd5a81a03de17f5514920a5c21377d827d742c4 Mon Sep 17 00:00:00 2001 From: Nathan Colosimo <110621881+NathanColosimo@users.noreply.github.com> Date: Thu, 20 Aug 2026 15:39:21 -0700 Subject: [PATCH 12/16] feat(world): observe streamed replay events --- .../prepare-streamed-replay-payloads.md | 4 +- packages/world-vercel/src/event-retry.ts | 8 +++ packages/world-vercel/src/events-v4.test.ts | 55 +++++++++++++++++- packages/world-vercel/src/events-v4.ts | 58 ++++++++++++++----- packages/world-vercel/src/events.test.ts | 40 ++++++++++++- packages/world-vercel/src/events.ts | 12 +++- packages/world/src/events.ts | 16 +++++ 7 files changed, 173 insertions(+), 20 deletions(-) diff --git a/.changeset/prepare-streamed-replay-payloads.md b/.changeset/prepare-streamed-replay-payloads.md index 9678641541..20facbfc94 100644 --- a/.changeset/prepare-streamed-replay-payloads.md +++ b/.changeset/prepare-streamed-replay-payloads.md @@ -1,5 +1,7 @@ --- "@workflow/core": patch +"@workflow/world": patch +"@workflow/world-vercel": patch --- -Prepare replay payloads as event frames arrive and reuse immutable primitive values across fresh workflow VMs. +Prepare replay payloads as validated event frames arrive and reuse immutable primitive values across fresh workflow VMs. diff --git a/packages/world-vercel/src/event-retry.ts b/packages/world-vercel/src/event-retry.ts index 30dbf6a97e..96dff4bcf9 100644 --- a/packages/world-vercel/src/event-retry.ts +++ b/packages/world-vercel/src/event-retry.ts @@ -88,6 +88,14 @@ export class EventPostResponseError extends Error { } } +/** Preserve an observer's original thrown value without letting it retry I/O. */ +export class EventObserverError extends EventPostResponseError { + constructor(readonly error: unknown) { + super('event observer failed', { cause: error }); + this.name = 'EventObserverError'; + } +} + export interface EventRetryPolicy { /** Whether a failed POST of this event type may be retried in-process. */ retryable: boolean; diff --git a/packages/world-vercel/src/events-v4.test.ts b/packages/world-vercel/src/events-v4.test.ts index 12314793b5..7b5f017970 100644 --- a/packages/world-vercel/src/events-v4.test.ts +++ b/packages/world-vercel/src/events-v4.test.ts @@ -374,6 +374,51 @@ describe('getWorkflowRunEventsV4 over HTTP', () => { agent.assertNoPendingInterceptors(); }); + it('does not retry an observer failure that resembles transport failure', async () => { + const agent = mockAgent(); + agent + .get(ORIGIN) + .intercept({ + path: '/api/v4/runs/wrun_1/events?remoteRefBehavior=resolve&returnAll=true', + method: 'GET', + }) + .reply( + 200, + Buffer.concat([ + encodeFrame( + { + eventId: 'evnt_1', + runId: 'wrun_1', + eventType: 'run_started', + createdAt: CREATED_AT, + }, + new Uint8Array() + ), + encodeFrame( + { _end: 1, next: 'eid:evnt_1', hasMore: false }, + new Uint8Array() + ), + ]), + { headers: { 'content-type': V4_FRAME_CONTENT_TYPE } } + ); + + const observerError = new WorkflowWorldError('observer failed', { + code: 'TRANSPORT', + }); + await expect( + getWorkflowRunEventsV4( + { + runId: 'wrun_1', + onEvent: () => { + throw observerError; + }, + }, + { token: 'test-token', dispatcher: agent } + ) + ).rejects.toBe(observerError); + agent.assertNoPendingInterceptors(); + }); + it('captures an explicit hasMore from the sentinel, independent of next', async () => { const agent = mockAgent(); @@ -1022,6 +1067,7 @@ describe('createWorkflowRunEventV4 over HTTP', () => { it('continues a truncated run_started stream after its last event', async () => { const agent = mockAgent(); + const observed: string[] = []; agent .get(ORIGIN) @@ -1081,7 +1127,8 @@ describe('createWorkflowRunEventV4 over HTTP', () => { const result = await createWorkflowRunStartedEventV4( { runId: 'wrun_1', specVersion: 5 }, - { token: 'test-token', dispatcher: agent } + { token: 'test-token', dispatcher: agent }, + (event) => observed.push(event.eventId) ); expect(result.events.map((event) => event.eventId)).toEqual([ @@ -1089,6 +1136,7 @@ describe('createWorkflowRunEventV4 over HTTP', () => { 'evnt_2', ]); expect(result.hasMore).toBe(false); + expect(observed).toEqual(['evnt_1', 'evnt_2']); agent.assertNoPendingInterceptors(); }); @@ -1162,6 +1210,7 @@ describe('createWorkflowRunEventV4 over HTTP', () => { it('retries a truncated continuation that produced no complete event', async () => { const agent = mockAgent(); + const observed: string[] = []; const continuationPath = '/api/v4/runs/wrun_1/events?cursor=eid%3Aevnt_1&remoteRefBehavior=resolve&returnAll=true'; @@ -1228,13 +1277,15 @@ describe('createWorkflowRunEventV4 over HTTP', () => { const result = await createWorkflowRunStartedEventV4( { runId: 'wrun_1', specVersion: 5 }, - { token: 'test-token', dispatcher: agent } + { token: 'test-token', dispatcher: agent }, + (event) => observed.push(event.eventId) ); expect(result.events.map((event) => event.eventId)).toEqual([ 'evnt_1', 'evnt_2', ]); + expect(observed).toEqual(['evnt_1', 'evnt_2']); expect(result.cursor).toBe('eid:evnt_2'); agent.assertNoPendingInterceptors(); }); diff --git a/packages/world-vercel/src/events-v4.ts b/packages/world-vercel/src/events-v4.ts index eea8772724..89385d5383 100644 --- a/packages/world-vercel/src/events-v4.ts +++ b/packages/world-vercel/src/events-v4.ts @@ -42,6 +42,7 @@ import { import { decode } from 'cbor-x'; import { z } from 'zod'; import { + EventObserverError, EventPostResponseError, isRetryableEventRequestError, } from './event-retry.js'; @@ -179,9 +180,10 @@ async function withV4ResponseBody( } catch (error) { if (!outcomeReported) { const incomplete = - error instanceof IncompleteFrameError || - error instanceof PartialEventStreamError || - isRecyclableTransportError(error); + !(error instanceof EventObserverError) && + (error instanceof IncompleteFrameError || + error instanceof PartialEventStreamError || + isRecyclableTransportError(error)); report(incomplete ? error : undefined); } throw error; @@ -850,11 +852,13 @@ async function decodeCreateEventResponse( export async function createWorkflowRunStartedEventV4( input: CreateEventV4InputBase, - config?: APIConfig + config?: APIConfig, + onEvent?: (event: Event) => void ) { const { responseHeaders, ...replay } = await postReplayLogEvent( { ...input, eventType: 'run_started' }, - config + config, + onEvent ); assert(replay.cursor, 'v4 createEvent: event stream missing cursor'); const maxEvents = MaxEventsHeaderSchema.safeParse( @@ -1277,7 +1281,8 @@ interface ReplayLog { */ export async function createHookReceivedPreloadEventV4( input: CreateEventV4InputBase, - config?: APIConfig + config?: APIConfig, + onEvent?: (event: Event) => void ): Promise< ReplayLog & { canonicalEventId: string | undefined; @@ -1286,7 +1291,8 @@ export async function createHookReceivedPreloadEventV4( > { const { responseHeaders, ...replay } = await postReplayLogEvent( { ...input, eventType: 'hook_received' }, - config + config, + onEvent ); const maxEvents = MaxEventsHeaderSchema.safeParse( responseHeaders.get(MAX_EVENTS_HEADER) @@ -1379,7 +1385,8 @@ const MAX_PARTIAL_EVENT_STREAM_RETRIES = 3; async function consumeEventFrameStream( response: Response, opName: string, - events: Event[] + events: Event[], + onEvent?: (event: Event) => void ): Promise<{ cursor: string | null; hasMore: boolean }> { try { for await (const frame of decodeFrames( @@ -1395,7 +1402,15 @@ async function consumeEventFrameStream( if (Object.keys(frame.meta).some((key) => key.startsWith('_'))) { throw new Error(`v4 ${opName}: unexpected control frame`); } - events.push(decodeEventFrame(frame)); + const event = decodeEventFrame(frame); + events.push(event); + if (onEvent) { + try { + onEvent(event); + } catch (error) { + throw new EventObserverError(error); + } + } } } catch (cause) { if (!(cause instanceof IncompleteFrameError)) throw cause; @@ -1416,7 +1431,8 @@ async function postReplayLogEvent( input: CreateEventV4InputBase & { eventType: 'run_started' | 'hook_received'; }, - config?: APIConfig + config?: APIConfig, + onEvent?: (event: Event) => void ): Promise { const events: Event[] = []; let responseHeaders: Headers | undefined; @@ -1426,7 +1442,12 @@ async function postReplayLogEvent( config, (response) => { responseHeaders = response.headers; - return consumeEventFrameStream(response, 'createEvent', events); + return consumeEventFrameStream( + response, + 'createEvent', + events, + onEvent + ); } ); assert(responseHeaders); @@ -1440,7 +1461,11 @@ async function postReplayLogEvent( try { const suffix = await getWorkflowRunEventsV4( - { runId: input.runId, pagination: { cursor: continuationCursor } }, + { + runId: input.runId, + pagination: { cursor: continuationCursor }, + onEvent, + }, config ); events.push(...suffix.data); @@ -1521,10 +1546,17 @@ export async function getWorkflowRunEventsV4( opName: 'listEvents', streamResponse: true, }, - (response) => consumeEventFrameStream(response, 'listEvents', events) + (response) => + consumeEventFrameStream( + response, + 'listEvents', + events, + params.onEvent + ) ); return { data: events, cursor: result.cursor, hasMore: result.hasMore }; } catch (error) { + if (error instanceof EventObserverError) throw error.error; const lastEvent = events.at(-1); if ( retries === MAX_PARTIAL_EVENT_STREAM_RETRIES || diff --git a/packages/world-vercel/src/events.test.ts b/packages/world-vercel/src/events.test.ts index c396ad9724..36c492a207 100644 --- a/packages/world-vercel/src/events.test.ts +++ b/packages/world-vercel/src/events.test.ts @@ -1,10 +1,11 @@ import { Buffer } from 'node:buffer'; import { gzipSync } from 'node:zlib'; +import { WorkflowWorldError } from '@workflow/errors'; import type { AnyEventRequest, CreateEventParams } from '@workflow/world'; import { decode, encode } from 'cbor-x'; import { ulid } from 'ulid'; import { MockAgent } from 'undici'; -import { describe, expect, it } from 'vitest'; +import { describe, expect, it, vi } from 'vitest'; import { EventPostResponseError } from './event-retry.js'; import { createWorkflowRunEvent, @@ -376,6 +377,43 @@ describe('createWorkflowRunEvent result contract', () => { agent.assertNoPendingInterceptors(); }); + it('does not repeat a POST when its event observer throws', async () => { + const agent = mockAgent(); + agent + .get(ORIGIN) + .intercept({ + path: '/api/v4/runs/wrun_1/events/run_started', + method: 'POST', + }) + .reply(200, runStartedResponse(), { + headers: { + 'content-type': V4_FRAME_CONTENT_TYPE, + 'x-wf-event-id': 'evnt_1', + 'x-wf-run-id': 'wrun_1', + 'x-wf-created-at': STARTED_AT.toISOString(), + 'x-wf-max-events': '10000', + }, + }); + + const observerError = new WorkflowWorldError('observer failed', { + code: 'TRANSPORT', + }); + const onEvent = vi.fn(() => { + throw observerError; + }); + + await expect( + createWorkflowRunEvent( + 'wrun_1', + { eventType: 'run_started', specVersion: 2 } as AnyEventRequest, + { onEvent }, + { token: 'test-token', dispatcher: agent } + ) + ).rejects.toBe(observerError); + expect(onEvent).toHaveBeenCalledOnce(); + agent.assertNoPendingInterceptors(); + }); + it('does not repeat a run_started POST when its continuation exhausts retries', async () => { const agent = mockAgent(); agent diff --git a/packages/world-vercel/src/events.ts b/packages/world-vercel/src/events.ts index d2903ff116..c7e6cb9d98 100644 --- a/packages/world-vercel/src/events.ts +++ b/packages/world-vercel/src/events.ts @@ -54,7 +54,7 @@ import { validateUlidTimestamp, type WorkflowRun, } from '@workflow/world'; -import { withEventPostRetry } from './event-retry.js'; +import { EventObserverError, withEventPostRetry } from './event-retry.js'; import { createHookReceivedPreloadEventV4, createWorkflowRunEventsBatchV4, @@ -622,6 +622,7 @@ export async function createWorkflowRunEvent( } return result as EventResult; } catch (err) { + if (err instanceof EventObserverError) throw err.error; // 404 on hook_disposed / hook_received → already-disposed hook. if ( isHookEventRequiringExistence(data.eventType) && @@ -736,7 +737,11 @@ async function createWorkflowRunEventInner( }; if (data.eventType === 'run_started' && !params?.skipPreload) { - const result = await createWorkflowRunStartedEventV4(input, config); + const result = await createWorkflowRunStartedEventV4( + input, + config, + params?.onEvent + ); const runCreated = result.events.find( (event) => event.eventType === 'run_created' ); @@ -781,7 +786,8 @@ async function createWorkflowRunEventInner( // default. const outcome = await createHookReceivedPreloadEventV4( { ...input, remoteRefBehavior: 'lazy' }, - config + config, + params.onEvent ); const { canonicalEventId, maxEvents, events, cursor, hasMore } = outcome; const canonicalEvent = events.find( diff --git a/packages/world/src/events.ts b/packages/world/src/events.ts index defec5e0f4..0aa9ce193d 100644 --- a/packages/world/src/events.ts +++ b/packages/world/src/events.ts @@ -941,6 +941,14 @@ export interface CreateEventParams { * `resumeHook()` must not set it. */ preloadEvents?: true; + /** + * Observe replay-preload events immediately after validation, in stream + * order. The callback is synchronous: it delays the next frame and therefore + * applies response backpressure. Recovery continues after the last observed + * event instead of replaying the accepted prefix. Throwing aborts the + * operation and is never treated as a transport failure or retried. + */ + onEvent?: (event: Event) => void; } /** @@ -1128,6 +1136,14 @@ export interface ListEventsParams { /** Omit `limit` to return every remaining event. */ pagination?: PaginationOptions; resolveData?: ResolveData; + /** + * Observe events immediately after validation, in stream order. The callback + * is synchronous: it delays the next frame and therefore applies response + * backpressure. Recovery continues after the last observed event instead of + * replaying the accepted prefix. Throwing aborts the operation and is never + * treated as a transport failure or retried. + */ + onEvent?: (event: Event) => void; } export interface ListEventsByCorrelationIdParams { From 1215838e0d06492d97facabd8c54866c38a88ce2 Mon Sep 17 00:00:00 2001 From: Nathan Colosimo <110621881+NathanColosimo@users.noreply.github.com> Date: Fri, 21 Aug 2026 09:07:51 -0700 Subject: [PATCH 13/16] fix(world-vercel): preserve replay observer failures --- packages/world-vercel/src/events-v4.test.ts | 19 +++--- packages/world-vercel/src/events-v4.ts | 19 +++++- packages/world-vercel/src/events.test.ts | 74 +++++++++++++++++++++ 3 files changed, 99 insertions(+), 13 deletions(-) diff --git a/packages/world-vercel/src/events-v4.test.ts b/packages/world-vercel/src/events-v4.test.ts index 7b5f017970..27c81e2e70 100644 --- a/packages/world-vercel/src/events-v4.test.ts +++ b/packages/world-vercel/src/events-v4.test.ts @@ -1208,11 +1208,13 @@ describe('createWorkflowRunEventV4 over HTTP', () => { agent.assertNoPendingInterceptors(); }); - it('retries a truncated continuation that produced no complete event', async () => { + it('resumes after an event observed during a truncated continuation', async () => { const agent = mockAgent(); const observed: string[] = []; const continuationPath = '/api/v4/runs/wrun_1/events?cursor=eid%3Aevnt_1&remoteRefBehavior=resolve&returnAll=true'; + const resumedPath = + '/api/v4/runs/wrun_1/events?cursor=eid%3Aevnt_2&remoteRefBehavior=resolve&returnAll=true'; agent .get(ORIGIN) @@ -1257,21 +1259,18 @@ describe('createWorkflowRunEventV4 over HTTP', () => { agent .get(ORIGIN) .intercept({ path: continuationPath, method: 'GET' }) - .reply(200, runStartedFrame.subarray(0, 4), { + .reply(200, runStartedFrame, { headers: { 'content-type': V4_FRAME_CONTENT_TYPE }, }); agent .get(ORIGIN) - .intercept({ path: continuationPath, method: 'GET' }) + .intercept({ path: resumedPath, method: 'GET' }) .reply( 200, - Buffer.concat([ - runStartedFrame, - encodeFrame( - { _end: 1, next: 'eid:evnt_2', hasMore: false }, - new Uint8Array() - ), - ]), + encodeFrame( + { _end: 1, next: 'eid:evnt_2', hasMore: false }, + new Uint8Array() + ), { headers: { 'content-type': V4_FRAME_CONTENT_TYPE } } ); diff --git a/packages/world-vercel/src/events-v4.ts b/packages/world-vercel/src/events-v4.ts index 89385d5383..e863e38438 100644 --- a/packages/world-vercel/src/events-v4.ts +++ b/packages/world-vercel/src/events-v4.ts @@ -1460,7 +1460,7 @@ async function postReplayLogEvent( const continuationCursor = `eid:${lastEvent.eventId}`; try { - const suffix = await getWorkflowRunEventsV4( + const suffix = await listWorkflowRunEventsV4( { runId: input.runId, pagination: { cursor: continuationCursor }, @@ -1476,6 +1476,7 @@ async function postReplayLogEvent( responseHeaders, }; } catch (cause) { + if (cause instanceof EventObserverError) throw cause; throw new EventPostResponseError( `v4 createEvent: replay continuation failed for run ${input.runId}`, { cause } @@ -1525,7 +1526,7 @@ function paginationToQuery( * `getWorkflowRunEvents` contract. A truncated full response resumes * after its last validated event instead of downloading accepted frames again. */ -export async function getWorkflowRunEventsV4( +async function listWorkflowRunEventsV4( params: ListEventsParams, config?: APIConfig ): Promise> { @@ -1556,7 +1557,7 @@ export async function getWorkflowRunEventsV4( ); return { data: events, cursor: result.cursor, hasMore: result.hasMore }; } catch (error) { - if (error instanceof EventObserverError) throw error.error; + if (error instanceof EventObserverError) throw error; const lastEvent = events.at(-1); if ( retries === MAX_PARTIAL_EVENT_STREAM_RETRIES || @@ -1570,6 +1571,18 @@ export async function getWorkflowRunEventsV4( } } +export async function getWorkflowRunEventsV4( + params: ListEventsParams, + config?: APIConfig +): Promise> { + try { + return await listWorkflowRunEventsV4(params, config); + } catch (error) { + if (error instanceof EventObserverError) throw error.error; + throw error; + } +} + /** * GET /api/v4/events?correlationId=...&runId=... * diff --git a/packages/world-vercel/src/events.test.ts b/packages/world-vercel/src/events.test.ts index 36c492a207..b41280e17b 100644 --- a/packages/world-vercel/src/events.test.ts +++ b/packages/world-vercel/src/events.test.ts @@ -414,6 +414,80 @@ describe('createWorkflowRunEvent result contract', () => { agent.assertNoPendingInterceptors(); }); + it('preserves an observer failure from a replay continuation', async () => { + const agent = mockAgent(); + agent + .get(ORIGIN) + .intercept({ + path: '/api/v4/runs/wrun_1/events/run_started', + method: 'POST', + }) + .reply( + 200, + encodeFrame( + { + eventId: 'evnt_0', + runId: 'wrun_1', + eventType: 'run_created', + createdAt: new Date('2026-06-09T23:59:59.000Z'), + specVersion: 2, + eventData: { + deploymentId: 'dpl_1', + workflowName: 'workflow', + }, + }, + new Uint8Array() + ), + { + headers: { + 'content-type': V4_FRAME_CONTENT_TYPE, + 'x-wf-max-events': '10000', + }, + } + ); + agent + .get(ORIGIN) + .intercept({ + path: '/api/v4/runs/wrun_1/events?cursor=eid%3Aevnt_0&remoteRefBehavior=resolve&returnAll=true', + method: 'GET', + }) + .reply( + 200, + encodeFrame( + { + eventId: 'evnt_1', + runId: 'wrun_1', + eventType: 'run_started', + createdAt: STARTED_AT, + specVersion: 2, + eventData: {}, + }, + new Uint8Array() + ), + { headers: { 'content-type': V4_FRAME_CONTENT_TYPE } } + ); + + const observerError = new WorkflowWorldError('observer failed', { + code: 'TRANSPORT', + }); + const observed: string[] = []; + const onEvent = vi.fn((event: { eventId: string }) => { + observed.push(event.eventId); + if (event.eventId === 'evnt_1') throw observerError; + }); + + await expect( + createWorkflowRunEvent( + 'wrun_1', + { eventType: 'run_started', specVersion: 2 } as AnyEventRequest, + { onEvent }, + { token: 'test-token', dispatcher: agent } + ) + ).rejects.toBe(observerError); + expect(observed).toEqual(['evnt_0', 'evnt_1']); + agent.assertNoPendingInterceptors(); + }); + it('does not repeat a run_started POST when its continuation exhausts retries', async () => { const agent = mockAgent(); agent From 68f866acc77c0b97b29e6841bcb35f741072117f Mon Sep 17 00:00:00 2001 From: Nathan Colosimo <110621881+NathanColosimo@users.noreply.github.com> Date: Fri, 21 Aug 2026 09:37:37 -0700 Subject: [PATCH 14/16] test(core): use centralized replay hydration --- .../core/src/serialization/compression-telemetry.test.ts | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/packages/core/src/serialization/compression-telemetry.test.ts b/packages/core/src/serialization/compression-telemetry.test.ts index 9ca73262fd..a3ce736689 100644 --- a/packages/core/src/serialization/compression-telemetry.test.ts +++ b/packages/core/src/serialization/compression-telemetry.test.ts @@ -117,10 +117,10 @@ describe('compression telemetry attributes', () => { ); recordedAttributes.length = 0; - await new ReplayPayloadCache(undefined).prepareEventPayload( + await new ReplayPayloadCache(undefined).getEventValue( 'evnt_test', - 'result', - data + data, + (prepared) => prepared ); const attrs = lastAttrs(); From 4988697495e8d97636e70b41c807bf51de0e6e18 Mon Sep 17 00:00:00 2001 From: Nathan Colosimo <110621881+NathanColosimo@users.noreply.github.com> Date: Fri, 21 Aug 2026 09:49:10 -0700 Subject: [PATCH 15/16] fix(core): compact buffered replay payloads --- packages/core/src/replay-payload-cache.test.ts | 6 ++++-- packages/core/src/replay-payload-cache.ts | 2 +- 2 files changed, 5 insertions(+), 3 deletions(-) diff --git a/packages/core/src/replay-payload-cache.test.ts b/packages/core/src/replay-payload-cache.test.ts index b2f192aad3..f868b981ba 100644 --- a/packages/core/src/replay-payload-cache.test.ts +++ b/packages/core/src/replay-payload-cache.test.ts @@ -1,3 +1,4 @@ +import { Buffer } from 'node:buffer'; import type { Event, WorkflowRun } from '@workflow/world'; import { assert, describe, expect, it, vi } from 'vitest'; import { importKey } from './encryption.js'; @@ -73,7 +74,7 @@ describe('ReplayPayloadCache', () => { }); it('compacts prepared bytes before retaining them', async () => { - const backing = new Uint8Array(64 * 1024); + const backing = Buffer.alloc(64 * 1024); const prepared = backing.subarray(1024, 2048); const cache = new ReplayPayloadCache( undefined, @@ -92,7 +93,8 @@ describe('ReplayPayloadCache', () => { 'data' in retained && retained.data instanceof Uint8Array ); - expect(retained.data).toEqual(prepared); + expect(Array.from(retained.data)).toEqual(Array.from(prepared)); + expect(retained.data).not.toBeInstanceOf(Buffer); expect(retained.data.buffer.byteLength).toBe(retained.data.byteLength); }); diff --git a/packages/core/src/replay-payload-cache.ts b/packages/core/src/replay-payload-cache.ts index dd40511ec1..283b55392b 100644 --- a/packages/core/src/replay-payload-cache.ts +++ b/packages/core/src/replay-payload-cache.ts @@ -29,7 +29,7 @@ type ReplayPayloadKey = string | typeof WORKFLOW_INPUT; function compactOwnedBytes(data: Uint8Array): Uint8Array { return data.byteOffset === 0 && data.byteLength === data.buffer.byteLength ? data - : data.slice(); + : new Uint8Array(data); } function isCacheablePrimitive(value: unknown): boolean { From c454a2cd5523e068c65724d7a0d0ab5af2bd51af Mon Sep 17 00:00:00 2001 From: Nathan Colosimo <110621881+NathanColosimo@users.noreply.github.com> Date: Fri, 21 Aug 2026 10:01:28 -0700 Subject: [PATCH 16/16] docs(core): describe actual replay preload fallbacks --- packages/core/src/runtime.ts | 13 ++++++------- 1 file changed, 6 insertions(+), 7 deletions(-) diff --git a/packages/core/src/runtime.ts b/packages/core/src/runtime.ts index 1336a8ca3f..135ee71d6b 100644 --- a/packages/core/src/runtime.ts +++ b/packages/core/src/runtime.ts @@ -2197,13 +2197,12 @@ export function workflowEntrypoint( ), }); } else { - // Successful write, no usable preload (CBOR response - // from an older server, a World that ignored the - // opt-in, a bounded hasMore page, or a preload that - // failed validation): take the generic run_started - // setup below. Its preload is loaded after this write - // committed, so the canonical hook_received is part - // of whatever log that setup reads — no splice + // Successful write, no usable preload (a World that + // ignored the opt-in, a bounded hasMore page, or a + // preload that failed validation): take the generic + // run_started setup below. Its preload is loaded after + // this write committed, so the canonical hook_received + // is part of whatever log that setup reads — no splice // needed. span?.setAttributes( Attribute.HookResumeSetupSource(