From 45c7bcdf12aea8911fd7e39c12e46d5bc365df4f Mon Sep 17 00:00:00 2001 From: Nathan Colosimo <110621881+NathanColosimo@users.noreply.github.com> Date: Fri, 17 Jul 2026 11:19:46 -0700 Subject: [PATCH 01/43] perf(core): retain workflow VM across inline steps Combines the retained-session architecture from #2984 with the env kill switch and loop-level single-VM test from #2966. - executeWorkflow with discriminated request/result types and a WorkflowSession state machine (running/suspended/failed/replay/completed) - EventsConsumer.append: only newly durable events feed the live VM - WORKFLOW_RETAINED_VM=0 kill switch (default on) - retained-vm-loop.test.ts: proves one VM per run and byte-identical output vs the from-scratch replay path --- .changeset/retain-workflow-vm.md | 6 + .../docs/v5/configuration/runtime-tuning.mdx | 7 + packages/core/src/events-consumer.test.ts | 27 ++ packages/core/src/events-consumer.ts | 7 +- packages/core/src/retained-vm-loop.test.ts | 178 +++++++++ packages/core/src/runtime.test.ts | 9 + packages/core/src/runtime.ts | 120 ++++-- packages/core/src/runtime/constants.ts | 17 + packages/core/src/telemetry.ts | 20 +- .../src/telemetry/semantic-conventions.ts | 5 + .../src/workflow-session-telemetry.test.ts | 69 ++++ packages/core/src/workflow.test.ts | 342 ++++++++++++++++- packages/core/src/workflow.ts | 356 +++++++++++++++--- 13 files changed, 1069 insertions(+), 94 deletions(-) create mode 100644 .changeset/retain-workflow-vm.md create mode 100644 packages/core/src/retained-vm-loop.test.ts create mode 100644 packages/core/src/workflow-session-telemetry.test.ts diff --git a/.changeset/retain-workflow-vm.md b/.changeset/retain-workflow-vm.md new file mode 100644 index 0000000000..37b874325e --- /dev/null +++ b/.changeset/retain-workflow-vm.md @@ -0,0 +1,6 @@ +--- +'@workflow/core': patch +'workflow': patch +--- + +Retain workflow execution across inline steps within one invocation. `WORKFLOW_RETAINED_VM=0` disables retention. diff --git a/docs/content/docs/v5/configuration/runtime-tuning.mdx b/docs/content/docs/v5/configuration/runtime-tuning.mdx index 34603add1e..f2f1a98e53 100644 --- a/docs/content/docs/v5/configuration/runtime-tuning.mdx +++ b/docs/content/docs/v5/configuration/runtime-tuning.mdx @@ -75,6 +75,13 @@ For example, a workflow can run a 10-minute inline step even with `WORKFLOW_REPL - Use only when step side effects are idempotent. - Set `0` or `false` to force it off, including the first-delivery fast path used by `WORKFLOW_TURBO`. +### `WORKFLOW_RETAINED_VM` + +- Default: enabled +- Keeps the suspended workflow VM alive across inline steps within one invocation, so each iteration of the inline loop appends only the newly written events instead of replaying the whole event log in a fresh VM. +- Suspensions involving hooks, waits, or attributes — and any replay divergence — always fall back to a full replay. +- Set `0` or `false` to replay from scratch in a fresh VM on every iteration. + ### `WORKFLOW_INLINE_OWNERSHIP` - Default: enabled diff --git a/packages/core/src/events-consumer.test.ts b/packages/core/src/events-consumer.test.ts index ecf828f728..3e320b16c7 100644 --- a/packages/core/src/events-consumer.test.ts +++ b/packages/core/src/events-consumer.test.ts @@ -45,6 +45,15 @@ describe('EventsConsumer', () => { expect(consumer.eventIndex).toBe(0); expect(consumer.callbacks).toEqual([]); }); + + it('should own its event array', () => { + const events = [createMockEvent()]; + const consumer = new EventsConsumer(events, defaultOptions); + + events.push(createMockEvent({ id: 'event-2' })); + + expect(consumer.events).toHaveLength(1); + }); }); describe('subscribe', () => { @@ -82,6 +91,24 @@ describe('EventsConsumer', () => { expect(callback).toHaveBeenCalledWith(event); expect(callback).toHaveBeenCalledTimes(1); }); + + it('should consume appended events', async () => { + const event = createMockEvent(); + const consumer = new EventsConsumer([], defaultOptions); + const callback = vi + .fn() + .mockImplementation((value: Event | null) => + value ? EventConsumerResult.Finished : EventConsumerResult.NotConsumed + ); + consumer.subscribe(callback); + await waitForNextTick(); + + consumer.append([event]); + await waitForNextTick(); + + expect(callback).toHaveBeenLastCalledWith(event); + expect(consumer.eventIndex).toBe(1); + }); }); describe('consume (implicit)', () => { diff --git a/packages/core/src/events-consumer.ts b/packages/core/src/events-consumer.ts index 4b7cf0742e..7b4afd6a9a 100644 --- a/packages/core/src/events-consumer.ts +++ b/packages/core/src/events-consumer.ts @@ -79,13 +79,18 @@ export class EventsConsumer { private unconsumedCheckVersion = 0; constructor(events: Event[], options: EventsConsumerOptions) { - this.events = events; + this.events = [...events]; this.eventIndex = 0; this.onConsumedEvent = options.onConsumedEvent; this.onUnconsumedEvent = options.onUnconsumedEvent; this.getPromiseQueue = options.getPromiseQueue; } + append(events: Event[]): void { + this.events.push(...events); + process.nextTick(this.consume); + } + /** * Registers a callback function to be called after an event has been consumed * by a different callback. The callback can return: diff --git a/packages/core/src/retained-vm-loop.test.ts b/packages/core/src/retained-vm-loop.test.ts new file mode 100644 index 0000000000..d30740ba69 --- /dev/null +++ b/packages/core/src/retained-vm-loop.test.ts @@ -0,0 +1,178 @@ +import { + type Event, + SPEC_VERSION_CURRENT, + type WorkflowRun, +} from '@workflow/world'; +import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest'; + +// Spy on VM-context construction while preserving the real implementation, so +// we can prove the retained path builds ONE VM for a whole run instead of one +// per replay iteration. workflow.ts imports `createContext` from this same +// module, so the spy observes its constructions too. +vi.mock('./vm/index.js', async (importActual) => { + const actual = await importActual(); + return { ...actual, createContext: vi.fn(actual.createContext) }; +}); + +const { createContext } = await import('./vm/index.js'); +const { registerStepFunction } = await import('./private.js'); +const { setWorld } = await import('./runtime/world.js'); +const { workflowEntrypoint } = await import('./runtime.js'); +const { dehydrateWorkflowArguments } = await import('./serialization.js'); + +vi.mock('@vercel/functions', () => ({ + waitUntil: vi.fn((p: Promise) => { + p.catch(() => {}); + }), +})); + +const createContextSpy = createContext as unknown as ReturnType; + +// Sequential two-step workflow: two replay-advancing suspensions before it +// completes — so a from-scratch replay builds the VM three times while the +// retained path builds it once. +const twoStepWorkflow = `const s1 = globalThis[Symbol.for("WORKFLOW_USE_STEP")]("r_s1"); + const s2 = globalThis[Symbol.for("WORKFLOW_USE_STEP")]("r_s2"); + async function workflow() { + const a = await s1(); + const b = await s2(); + return a + b; + } + globalThis.__private_workflows = new Map([["workflow", workflow]]);`; + +registerStepFunction('r_s1', async () => 10); +registerStepFunction('r_s2', async () => 20); + +// Drive the full workflow handler over a stateful (dynamic) event log so the +// inline loop makes real progress across its own writes, exactly like a World. +// Non-turbo (no runInput, attempt 2) to keep the path simple and deterministic. +async function drive(runId: string) { + const run: WorkflowRun = { + runId, + workflowName: 'workflow', + status: 'running', + input: await dehydrateWorkflowArguments([], runId, undefined, []), + createdAt: new Date('2024-01-01T00:00:00.000Z'), + updatedAt: new Date('2024-01-01T00:00:00.000Z'), + startedAt: new Date('2024-01-01T00:00:00.000Z'), + deploymentId: 'test-deployment', + }; + const events: Event[] = []; + const createdEvents: any[] = []; + let seq = 0; + + const eventsCreate = vi.fn(async (_runId: string, data: any) => { + createdEvents.push(data); + if (data.eventType === 'run_started') { + return { run, events }; + } + const event = { + eventId: `e-${++seq}`, + runId, + createdAt: new Date(), + ...data, + } as Event; + events.push(event); + // step_started returns a running step entity so executeStep proceeds to + // run the body and write step_completed. + if (data.eventType === 'step_started') { + const d = data.eventData as { stepName?: string; input?: unknown }; + return { + event, + step: { + runId, + stepId: data.correlationId, + stepName: d?.stepName, + status: 'running' as const, + attempt: 1, + input: d?.input, + startedAt: new Date(), + createdAt: new Date(), + updatedAt: new Date(), + }, + ...(d?.input !== undefined ? { stepCreated: true } : {}), + }; + } + return { event }; + }); + + setWorld({ + specVersion: SPEC_VERSION_CURRENT, + createQueueHandler: vi.fn( + (_p: string, handler: (m: unknown, md: unknown) => Promise) => + async () => { + await handler( + { runId, requestedAt: new Date('2024-01-01T00:00:00.000Z') }, + { + requestId: 'req_retained', + attempt: 2, + queueName: '__wkf_workflow_workflow', + messageId: 'msg_retained', + } + ); + return new Response(null, { status: 204 }); + } + ), + events: { + create: eventsCreate, + list: vi.fn(async () => ({ + data: [...events], + hasMore: false, + cursor: 'cursor_retained', + })), + }, + runs: { get: vi.fn(async () => run) }, + queue: vi.fn(async () => ({ messageId: null })), + getEncryptionKeyForRun: vi.fn(async () => undefined), + } as any); + + await workflowEntrypoint(twoStepWorkflow)( + new Request('https://example.test') + ); + + const runCompleted = createdEvents.find( + (e) => e.eventType === 'run_completed' + ); + return { + vmBuilds: createContextSpy.mock.calls.length, + output: runCompleted?.eventData?.output as Uint8Array | undefined, + }; +} + +describe('retained VM through the inline replay loop', () => { + beforeEach(() => { + createContextSpy.mockClear(); + }); + afterEach(() => { + delete process.env.WORKFLOW_RETAINED_VM; + setWorld(undefined); + vi.clearAllMocks(); + }); + + it('rebuilds the VM once per replay under the kill switch (WORKFLOW_RETAINED_VM=0)', async () => { + process.env.WORKFLOW_RETAINED_VM = '0'; + const { vmBuilds, output } = await drive('wrun_retained_off'); + expect(output).toBeInstanceOf(Uint8Array); + // A 2-step sequential workflow suspends twice then completes → >1 replay, + // each building a fresh VM. + expect(vmBuilds).toBeGreaterThan(1); + }); + + it('builds the VM once and resumes it when retention is ON (the default)', async () => { + // Baseline via the kill switch, to compare the dehydrated result against. + process.env.WORKFLOW_RETAINED_VM = '0'; + const off = await drive('wrun_retained_baseline_off'); + createContextSpy.mockClear(); + + // Unset → retention is ON by default. + delete process.env.WORKFLOW_RETAINED_VM; + const on = await drive('wrun_retained_on'); + + expect(on.output).toBeInstanceOf(Uint8Array); + // The whole run is served by a single VM: built once on the first pass, + // resumed (not rebuilt) for every subsequent step. + expect(on.vmBuilds).toBe(1); + // And it produces the identical dehydrated result as the replay path. + expect(on.output).toEqual(off.output); + }); +}); diff --git a/packages/core/src/runtime.test.ts b/packages/core/src/runtime.test.ts index aafec00075..1e229e2fec 100644 --- a/packages/core/src/runtime.test.ts +++ b/packages/core/src/runtime.test.ts @@ -11,6 +11,7 @@ import { } from '@workflow/world'; import { ulid } from 'ulid'; import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest'; +import { runtimeLogger } from './logger.js'; import { registerStepFunction } from './private.js'; import { REPLAY_DIVERGENCE_MAX_RETRIES } from './runtime/constants.js'; import { setWorld } from './runtime/world.js'; @@ -702,6 +703,9 @@ describe('workflowEntrypoint replay guards', () => { }); it('replays attribute events before executing a step that loses the same race', async () => { + const debug = vi + .spyOn(runtimeLogger, 'debug') + .mockImplementation(() => undefined); const ops: Promise[] = []; const workflowRun: WorkflowRun = { runId: 'wrun_attribute_step_race', @@ -760,6 +764,11 @@ describe('workflowEntrypoint replay guards', () => { expect(createdEvents).not.toContainEqual( expect.objectContaining({ eventType: 'step_started' }) ); + const executionModes = debug.mock.calls + .filter(([message]) => message === 'Starting workflow execution') + .map(([, context]) => context?.executionMode); + expect(executionModes).toEqual(['replay', 'replay']); + debug.mockRestore(); }); it('fails the run when the World rejects an attr_set event as invalid', async () => { diff --git a/packages/core/src/runtime.ts b/packages/core/src/runtime.ts index 8780818421..a085c44884 100644 --- a/packages/core/src/runtime.ts +++ b/packages/core/src/runtime.ts @@ -40,6 +40,7 @@ import { getReplayDivergenceMaxRetries, isInlineOwnershipEnabled, isTurboEnabled, + isVmRetentionEnabled, } from './runtime/constants.js'; import { getQueueOverhead, @@ -91,7 +92,11 @@ import { } from './telemetry.js'; import { getErrorName, getErrorStack, normalizeUnknownError } from './types.js'; import { buildWorkflowSuspensionMessage } from './util.js'; -import { runWorkflow } from './workflow.js'; +import { + executeWorkflow, + type WorkflowExecutionResult, + type WorkflowSession, +} from './workflow.js'; export type { Event, WorkflowRun }; export { WorkflowSuspension } from './global.js'; @@ -311,6 +316,27 @@ function openHookAndWaitState(events: Event[]): { return { openHook, openWait }; } +/** + * Retain only pure step boundaries with no out-of-band continuation source. + * Attributes require replay; hooks and waits can wake another invocation. + * `WORKFLOW_RETAINED_VM=0` disables retention entirely. + */ +function canRetainWorkflowSession( + suspension: WorkflowSuspension, + events: Event[] +): boolean { + return ( + isVmRetentionEnabled() && + suspension.stepCount > 0 && + suspension.hookCount === 0 && + suspension.waitCount === 0 && + suspension.attributeCount === 0 && + suspension.hookDisposedCount === 0 && + suspension.abortCount === 0 && + !hasOpenHookOrWait(events) + ); +} + /** * Creates a single route which handles workflow execution requests, * executing steps inline when possible to reduce function invocations @@ -1105,6 +1131,13 @@ export function workflowEntrypoint( encryptionKey ); + let workflowExecution: + | { readonly type: 'replay' } + | { + readonly type: 'retained'; + readonly session: WorkflowSession; + } = { type: 'replay' }; + // Main replay loop // biome-ignore lint/correctness/noConstantCondition: intentional loop while (true) { @@ -1419,37 +1452,76 @@ export function workflowEntrypoint( // point and the inline executeStep mutates eventsCursor. preInlineWriteCursor = eventsCursor; - // Replay workflow - runtimeLogger.debug('Starting workflow replay', { + let executionMode = workflowExecution.type; + runtimeLogger.debug('Starting workflow execution', { workflowRunId: runId, loopIteration, eventCount: events.length, + executionMode, }); replayStart = Date.now(); - // Start every missing decrypt/decompress operation before - // VM setup. Web Crypto work can overlap bundle evaluation; - // consumers still deserialize and resolve in event order. - const payloadPrewarm = replayPayloadCache.prewarm( - workflowRun, - events - ); - const result = await runWorkflow( - workflowCode, - workflowRun, - events, - encryptionKey, - replayPayloadCache, - // Turbo: the end-of-run drain inside runWorkflow commits - // fire-and-forget `*_created` events before the terminal - // `awaitRunReady()` below, so gate those writes on the - // backgrounded run_started too. Undefined outside turbo. - runReadyBarrier - ); - await payloadPrewarm; - runtimeLogger.debug('Workflow replay completed', { + let workflowResult: WorkflowExecutionResult = { + type: 'replay', + }; + if (workflowExecution.type === 'retained') { + workflowResult = await executeWorkflow({ + type: 'resume', + session: workflowExecution.session, + events, + }); + } + + if (workflowResult.type === 'replay') { + executionMode = 'replay'; + workflowExecution = { type: 'replay' }; + // Start every missing decrypt/decompress operation + // before VM setup. Web Crypto work can overlap bundle + // evaluation; consumers still deserialize and resolve + // in event order. + const payloadPrewarm = replayPayloadCache.prewarm( + workflowRun, + events + ); + workflowResult = await executeWorkflow({ + type: 'replay', + workflowCode, + workflowRun, + events, + encryptionKey, + replayPayloadCache, + // Turbo: the end-of-run drain inside workflow + // execution commits fire-and-forget `*_created` + // events before the terminal `awaitRunReady()` below. + runReadyBarrier, + }); + await payloadPrewarm; + } + + if (workflowResult.type === 'suspended') { + workflowExecution = canRetainWorkflowSession( + workflowResult.suspension, + events + ) + ? { + type: 'retained', + session: workflowResult.session, + } + : { type: 'replay' }; + throw workflowResult.suspension; + } + + if (workflowResult.type === 'replay') { + throw new Error( + 'Invariant violation: fresh workflow execution requested another replay' + ); + } + + const result = workflowResult.output; + runtimeLogger.debug('Workflow execution completed', { workflowRunId: runId, loopIteration, replayMs: Date.now() - replayStart, + executionMode, }); // Workflow completed. Send the snapshot but do NOT diff --git a/packages/core/src/runtime/constants.ts b/packages/core/src/runtime/constants.ts index 8db7a3f723..b72bed773b 100644 --- a/packages/core/src/runtime/constants.ts +++ b/packages/core/src/runtime/constants.ts @@ -275,6 +275,23 @@ export function isTurboEnabled(): boolean { return !(raw === '0' || raw.toLowerCase() === 'false'); } +/** + * Whether the inline loop retains a suspended workflow VM across inline steps + * within one invocation (default ON). When on, a step-only suspension keeps + * the live VM, event consumer, and hydrated state alive, and the next loop + * iteration appends only the newly durable events instead of rebuilding the + * `vm.Context` and replaying the whole event log. Non-step suspensions and + * replay divergence always fall back to the ordinary durable replay path. + * + * `WORKFLOW_RETAINED_VM=0` (or `false`) is the kill switch: every iteration + * replays from scratch in a fresh VM, matching the pre-retention behavior. + */ +export function isVmRetentionEnabled(): boolean { + const raw = process.env.WORKFLOW_RETAINED_VM; + if (raw === undefined || raw === '') return true; + return !(raw === '0' || raw.toLowerCase() === 'false'); +} + /** * Whether inline step ownership is enabled (default ON). When on, the lazy * `step_started` that creates an inline step records the owning queue diff --git a/packages/core/src/telemetry.ts b/packages/core/src/telemetry.ts index da69d2756d..0c3a32f9d5 100644 --- a/packages/core/src/telemetry.ts +++ b/packages/core/src/telemetry.ts @@ -249,7 +249,10 @@ export async function trace( code: otel.SpanStatusCode.ERROR, message: (e as Error).message, }); - applyWorkflowSuspensionToSpan(e, otel, span); + if (WorkflowSuspension.is(e)) { + span.setStatus({ code: otel.SpanStatusCode.OK }); + applyWorkflowSuspensionToSpan(e, span); + } throw e; } finally { span.end(); @@ -275,19 +278,12 @@ export async function recordElapsedSpan( } /** - * Applies workflow suspension attributes to the given span if the error is a WorkflowSuspension - * which is technically not an error, but an algebraic effect indicating suspension. + * Applies the workflow suspension algebraic effect to an active span. */ -function applyWorkflowSuspensionToSpan( - error: unknown, - otel: typeof api, +export function applyWorkflowSuspensionToSpan( + error: WorkflowSuspension, span: api.Span -) { - if (!error || !WorkflowSuspension.is(error)) { - return; - } - - span.setStatus({ code: otel.SpanStatusCode.OK }); +): void { span.setAttributes({ ...Attr.WorkflowSuspensionState('suspended'), ...Attr.WorkflowSuspensionStepCount(error.stepCount), diff --git a/packages/core/src/telemetry/semantic-conventions.ts b/packages/core/src/telemetry/semantic-conventions.ts index bcc6d1dabf..0793b26d87 100644 --- a/packages/core/src/telemetry/semantic-conventions.ts +++ b/packages/core/src/telemetry/semantic-conventions.ts @@ -77,6 +77,11 @@ export const WorkflowEventsCount = SemanticConvention( 'workflow.events.count' ); +/** Whether workflow execution starts with replay or resumes a retained VM */ +export const WorkflowExecutionMode = SemanticConvention<'replay' | 'retained'>( + 'workflow.execution.mode' +); + /** Number of arguments passed to the workflow */ export const WorkflowArgumentsCount = SemanticConvention( 'workflow.arguments.count' diff --git a/packages/core/src/workflow-session-telemetry.test.ts b/packages/core/src/workflow-session-telemetry.test.ts new file mode 100644 index 0000000000..49058a4585 --- /dev/null +++ b/packages/core/src/workflow-session-telemetry.test.ts @@ -0,0 +1,69 @@ +import { trace as otelTrace } from '@opentelemetry/api'; +import { + BasicTracerProvider, + InMemorySpanExporter, + SimpleSpanProcessor, +} from '@opentelemetry/sdk-trace-base'; +import type { WorkflowRun } from '@workflow/world'; +import { afterAll, afterEach, beforeAll, describe, expect, it } from 'vitest'; +import { ReplayPayloadCache } from './replay-payload-cache.js'; +import { dehydrateWorkflowArguments } from './serialization.js'; +import { executeWorkflow } from './workflow.js'; + +const exporter = new InMemorySpanExporter(); +const provider = new BasicTracerProvider(); + +beforeAll(() => { + provider.addSpanProcessor(new SimpleSpanProcessor(exporter)); + otelTrace.setGlobalTracerProvider(provider); +}); + +afterAll(async () => { + await provider.shutdown(); + otelTrace.disable(); +}); + +afterEach(() => { + exporter.reset(); +}); + +describe('retained workflow telemetry', () => { + it('records suspension attributes on workflow.run spans', async () => { + const runId = 'wrun_retained_telemetry'; + const workflowRun: WorkflowRun = { + runId, + workflowName: 'workflow', + status: 'running', + input: await dehydrateWorkflowArguments([], runId, undefined, []), + createdAt: new Date('2024-01-01T00:00:00.000Z'), + updatedAt: new Date('2024-01-01T00:00:00.000Z'), + startedAt: new Date('2024-01-01T00:00:00.000Z'), + deploymentId: 'test-deployment', + }; + const workflowCode = ` + const step = globalThis[Symbol.for("WORKFLOW_USE_STEP")]("step"); + async function workflow() { await step(); } + globalThis.__private_workflows = new Map([["workflow", workflow]]);`; + + const result = await executeWorkflow({ + type: 'replay', + workflowCode, + workflowRun, + events: [], + encryptionKey: undefined, + replayPayloadCache: new ReplayPayloadCache(undefined), + }); + expect(result.type).toBe('suspended'); + + const span = exporter + .getFinishedSpans() + .find((candidate) => candidate.name === 'workflow.run workflow'); + expect(span?.attributes).toMatchObject({ + 'workflow.execution.mode': 'replay', + 'workflow.suspension.state': 'suspended', + 'workflow.suspension.step_count': 1, + 'workflow.suspension.hook_count': 0, + 'workflow.suspension.wait_count': 0, + }); + }); +}); diff --git a/packages/core/src/workflow.test.ts b/packages/core/src/workflow.test.ts index fc21423518..66cc295e27 100644 --- a/packages/core/src/workflow.test.ts +++ b/packages/core/src/workflow.test.ts @@ -6,6 +6,7 @@ import { monotonicFactory } from 'ulid'; import { afterEach, assert, describe, expect, it, vi } from 'vitest'; import { DEFERRED_CHECK_DELAY_MS } from './events-consumer.js'; import type { WorkflowSuspension } from './global.js'; +import { ReplayPayloadCache } from './replay-payload-cache.js'; import { setWorld } from './runtime/world.js'; import { dehydrateStepReturnValue, @@ -13,7 +14,7 @@ import { hydrateWorkflowReturnValue, } from './serialization.js'; import { createContext } from './vm/index.js'; -import { runWorkflow } from './workflow.js'; +import { executeWorkflow, runWorkflow } from './workflow.js'; // No encryption key = encryption disabled const noEncryptionKey = undefined; @@ -221,6 +222,345 @@ describe('runWorkflow', () => { ).toEqual(3); }); + it('should retain workflow execution across sequential step suspensions', async () => { + const ops: Promise[] = []; + const workflowRun: WorkflowRun = { + runId: 'wrun_retained', + workflowName: 'workflow', + status: 'running', + input: await dehydrateWorkflowArguments( + [], + 'wrun_retained', + noEncryptionKey, + ops + ), + createdAt: new Date('2024-01-01T00:00:00.000Z'), + updatedAt: new Date('2024-01-01T00:00:00.000Z'), + startedAt: new Date('2024-01-01T00:00:00.000Z'), + deploymentId: 'test-deployment', + }; + const workflowCode = ` + const add = globalThis[Symbol.for("WORKFLOW_USE_STEP")]("add"); + async function workflow() { + console.log("retained:entered"); + const first = await add(1, 2); + console.log("retained:continued"); + const second = await add(first, 3); + return second; + } + ${getWorkflowTransformCode('workflow')}`; + const log = vi.spyOn(console, 'log').mockImplementation(() => undefined); + const events: Event[] = []; + let eventNumber = 0; + const appendStepEvents = async ( + correlationId: string, + result: number + ): Promise => { + const createdAt = new Date(`2024-01-01T00:00:0${eventNumber + 1}.000Z`); + events.push( + { + eventId: `event-${eventNumber++}`, + runId: workflowRun.runId, + eventType: 'step_created', + correlationId, + eventData: { stepName: 'add' }, + createdAt, + }, + { + eventId: `event-${eventNumber++}`, + runId: workflowRun.runId, + eventType: 'step_started', + correlationId, + eventData: { stepName: 'add' }, + createdAt, + }, + { + eventId: `event-${eventNumber++}`, + runId: workflowRun.runId, + eventType: 'step_completed', + correlationId, + eventData: { + stepName: 'add', + result: await dehydrateStepReturnValue( + result, + workflowRun.runId, + noEncryptionKey, + ops + ), + }, + createdAt, + } + ); + }; + + const first = await executeWorkflow({ + type: 'replay', + workflowCode, + workflowRun, + events, + encryptionKey: noEncryptionKey, + replayPayloadCache: new ReplayPayloadCache(noEncryptionKey), + }); + assert(first.type === 'suspended'); + const firstStep = first.suspension.steps[0]; + assert(firstStep?.type === 'step'); + + await appendStepEvents(firstStep.correlationId, 3); + + const second = await executeWorkflow({ + type: 'resume', + session: first.session, + events, + }); + assert(second.type === 'suspended'); + expect(second.session).toBe(first.session); + const secondStep = second.suspension.steps[0]; + assert(secondStep?.type === 'step'); + + await appendStepEvents(secondStep.correlationId, 6); + + const completed = await executeWorkflow({ + type: 'resume', + session: second.session, + events, + }); + assert(completed.type === 'completed'); + + expect( + await hydrateWorkflowReturnValue( + completed.output as any, + workflowRun.runId, + noEncryptionKey, + ops + ) + ).toBe(6); + expect( + log.mock.calls + .map(([message]) => message) + .filter((message) => String(message).startsWith('retained:')) + ).toEqual(['retained:entered', 'retained:continued']); + log.mockRestore(); + }); + + it('returns a workflow that completes while its retained session is suspended', async () => { + const ops: Promise[] = []; + const workflowRun: WorkflowRun = { + runId: 'wrun_retained_async_completion', + workflowName: 'workflow', + status: 'running', + input: await dehydrateWorkflowArguments( + [], + 'wrun_retained_async_completion', + noEncryptionKey, + ops + ), + createdAt: new Date('2024-01-01T00:00:00.000Z'), + updatedAt: new Date('2024-01-01T00:00:00.000Z'), + startedAt: new Date('2024-01-01T00:00:00.000Z'), + deploymentId: 'test-deployment', + }; + const workflowCode = ` + const step = globalThis[Symbol.for("WORKFLOW_USE_STEP")]("step"); + async function digestRepeatedly() { + const bytes = new Uint8Array(8 * 1024 * 1024); + for (let index = 0; index < 8; index++) { + await crypto.subtle.digest("SHA-256", bytes); + } + } + async function workflow() { + await Promise.race([step(), digestRepeatedly()]); + console.log("retained:digest-completed"); + return "digest completed"; + } + ${getWorkflowTransformCode('workflow')}`; + const events: Event[] = [ + { + eventId: 'event-run-created', + runId: workflowRun.runId, + eventType: 'run_created', + createdAt: new Date('2024-01-01T00:00:00.000Z'), + }, + ]; + const log = vi.spyOn(console, 'log').mockImplementation(() => undefined); + + const suspended = await executeWorkflow({ + type: 'replay', + workflowCode, + workflowRun, + events, + encryptionKey: noEncryptionKey, + replayPayloadCache: new ReplayPayloadCache(noEncryptionKey), + }); + assert(suspended.type === 'suspended'); + const pendingStep = suspended.suspension.steps[0]; + assert(pendingStep?.type === 'step'); + + const rewrittenSession = await executeWorkflow({ + type: 'replay', + workflowCode, + workflowRun, + events, + encryptionKey: noEncryptionKey, + replayPayloadCache: new ReplayPayloadCache(noEncryptionKey), + }); + assert(rewrittenSession.type === 'suspended'); + + await vi.waitFor( + () => { + expect(log).toHaveBeenCalledTimes(2); + }, + { timeout: 10_000, interval: 10 } + ); + const rewrittenEvents = [ + { ...events[0], eventId: 'rewritten-event' } as Event, + { ...events[0], eventId: 'new-event' } as Event, + ]; + expect( + await executeWorkflow({ + type: 'resume', + session: rewrittenSession.session, + events: rewrittenEvents, + }) + ).toEqual({ type: 'replay' }); + + const createdAt = new Date('2024-01-01T00:00:01.000Z'); + events.push( + { + eventId: 'event-step-created', + runId: workflowRun.runId, + eventType: 'step_created', + correlationId: pendingStep.correlationId, + eventData: { stepName: pendingStep.stepName }, + createdAt, + }, + { + eventId: 'event-step-started', + runId: workflowRun.runId, + eventType: 'step_started', + correlationId: pendingStep.correlationId, + eventData: { stepName: pendingStep.stepName }, + createdAt, + }, + { + eventId: 'event-step-completed', + runId: workflowRun.runId, + eventType: 'step_completed', + correlationId: pendingStep.correlationId, + eventData: { + stepName: pendingStep.stepName, + result: await dehydrateStepReturnValue( + undefined, + workflowRun.runId, + noEncryptionKey, + ops + ), + }, + createdAt, + } + ); + + const completed = await executeWorkflow({ + type: 'resume', + session: suspended.session, + events, + }); + assert(completed.type === 'completed'); + expect( + await executeWorkflow({ + type: 'resume', + session: rewrittenSession.session, + events, + }) + ).toEqual({ type: 'replay' }); + expect( + await hydrateWorkflowReturnValue( + completed.output as any, + workflowRun.runId, + noEncryptionKey, + ops + ) + ).toBe('digest completed'); + log.mockRestore(); + }); + + it('does not drain operations from a discarded retained session', async () => { + const ops: Promise[] = []; + const workflowRun: WorkflowRun = { + runId: 'wrun_retained_discarded', + workflowName: 'workflow', + status: 'running', + input: await dehydrateWorkflowArguments( + [], + 'wrun_retained_discarded', + noEncryptionKey, + ops + ), + createdAt: new Date('2024-01-01T00:00:00.000Z'), + updatedAt: new Date('2024-01-01T00:00:00.000Z'), + startedAt: new Date('2024-01-01T00:00:00.000Z'), + deploymentId: 'test-deployment', + }; + const workflowCode = ` + const step = globalThis[Symbol.for("WORKFLOW_USE_STEP")]("step"); + const setAttributes = globalThis[Symbol.for("WORKFLOW_SET_ATTRIBUTES")]; + async function digestRepeatedly() { + const bytes = new Uint8Array(8 * 1024 * 1024); + for (let index = 0; index < 8; index++) { + await crypto.subtle.digest("SHA-256", bytes); + } + } + async function workflow() { + await Promise.race([step(), digestRepeatedly()]); + void setAttributes([{ key: "stale", value: true }]); + console.log("retained:discarded-completed"); + return "discarded"; + } + ${getWorkflowTransformCode('workflow')}`; + const create = vi.fn(); + const log = vi.spyOn(console, 'log').mockImplementation(() => undefined); + setWorld({ + specVersion: SPEC_VERSION_CURRENT, + events: { create }, + streams: { write: vi.fn(), close: vi.fn() }, + } as any); + + const suspended = await executeWorkflow({ + type: 'replay', + workflowCode, + workflowRun, + events: [], + encryptionKey: noEncryptionKey, + replayPayloadCache: new ReplayPayloadCache(noEncryptionKey), + }); + assert(suspended.type === 'suspended'); + + await vi.waitFor( + () => { + expect(log).toHaveBeenCalledWith('retained:discarded-completed'); + }, + { timeout: 10_000, interval: 10 } + ); + await new Promise((resolve) => setTimeout(resolve, 250)); + + expect( + await executeWorkflow({ + type: 'resume', + session: suspended.session, + events: [ + { + eventId: 'event-run-created', + runId: workflowRun.runId, + eventType: 'run_created', + createdAt: new Date('2024-01-01T00:00:00.000Z'), + }, + ], + }) + ).toEqual({ type: 'replay' }); + expect(create).not.toHaveBeenCalled(); + setWorld(undefined); + log.mockRestore(); + }); + it('regenerates step correlation IDs independent of startedAt (turbo replay-stability)', async () => { // Turbo's first delivery synthesizes `startedAt` from the local clock, // while later (non-turbo) deliveries load the server-canonical `startedAt`. diff --git a/packages/core/src/workflow.ts b/packages/core/src/workflow.ts index 4595f6a17c..1fe6668470 100644 --- a/packages/core/src/workflow.ts +++ b/packages/core/src/workflow.ts @@ -4,7 +4,11 @@ import { WorkflowNotRegisteredError, WorkflowRuntimeError, } from '@workflow/errors'; -import { createWorkflowBaseUrl, withResolvers } from '@workflow/utils'; +import { + createWorkflowBaseUrl, + type PromiseWithResolvers, + withResolvers, +} from '@workflow/utils'; import { parseWorkflowName } from '@workflow/utils/parse-name'; import type { Event, WorkflowRun } from '@workflow/world'; import { SPEC_VERSION_SUPPORTS_COMPRESSION } from '@workflow/world'; @@ -36,7 +40,7 @@ import { WORKFLOW_USE_STEP, } from './symbols.js'; import * as Attribute from './telemetry/semantic-conventions.js'; -import { trace } from './telemetry.js'; +import { applyWorkflowSuspensionToSpan, trace } from './telemetry.js'; import { getWorkflowRunStreamId } from './util.js'; import { createContext } from './vm/index.js'; import { runCachedWorkflowScript } from './vm/script-cache.js'; @@ -133,6 +137,142 @@ async function drainPendingQueueItems( } } +interface WorkflowCompletion { + readonly output: unknown; + readonly resultType: string; +} + +type WorkflowSessionState = + | { + readonly type: 'running'; + readonly interruption: PromiseWithResolvers; + } + | { readonly type: 'suspended'; readonly suspension: WorkflowSuspension } + | { readonly type: 'failed'; readonly error: Error } + | { readonly type: 'replay' } + | { readonly type: 'completed' }; + +function isSameSuspensionBoundary( + previous: WorkflowSuspension, + next: WorkflowSuspension +): boolean { + return ( + previous.stepCount === next.stepCount && + previous.hookCount === next.hookCount && + previous.waitCount === next.waitCount && + previous.attributeCount === next.attributeCount && + previous.hookDisposedCount === next.hookDisposedCount && + previous.abortCount === next.abortCount && + previous.steps.length === next.steps.length && + previous.steps.every((item, index) => item === next.steps[index]) + ); +} + +function updateSuspendedSession( + suspension: WorkflowSuspension, + error: Error +): WorkflowSessionState { + if (!WorkflowSuspension.is(error)) return { type: 'failed', error }; + return isSameSuspensionBoundary(suspension, error) + ? { type: 'suspended', suspension } + : { type: 'replay' }; +} + +export interface WorkflowSession { + readonly workflowRun: WorkflowRun; + readonly argumentCount: number; + resume(events: Event[]): WorkflowSessionResumeResult; +} + +type WorkflowSessionResumeResult = + | { + readonly type: 'resumed'; + readonly execution: Promise; + } + | { readonly type: 'replay' }; + +interface WorkflowSessionOptions { + readonly workflowCode: string; + readonly workflowRun: WorkflowRun; + readonly events: Event[]; + readonly encryptionKey: CryptoKey | undefined; + readonly replayPayloadCache: ReplayPayloadCache; + readonly runReadyBarrier?: Promise; +} + +type WorkflowExecutionRequest = + | ({ readonly type: 'replay' } & WorkflowSessionOptions) + | { + readonly type: 'resume'; + readonly session: WorkflowSession; + readonly events: Event[]; + }; + +export type WorkflowExecutionResult = + | { readonly type: 'replay' } + | { readonly type: 'completed'; readonly output: unknown } + | { + readonly type: 'suspended'; + readonly suspension: WorkflowSuspension; + readonly session: WorkflowSession; + }; + +export async function executeWorkflow( + request: WorkflowExecutionRequest +): Promise { + const workflowRun = + request.type === 'replay' + ? request.workflowRun + : request.session.workflowRun; + const mode = request.type === 'replay' ? 'replay' : 'retained'; + + return trace(`workflow.run ${workflowRun.workflowName}`, async (span) => { + span?.setAttributes({ + ...Attribute.WorkflowName(workflowRun.workflowName), + ...Attribute.WorkflowRunId(workflowRun.runId), + ...Attribute.WorkflowRunStatus(workflowRun.status), + ...Attribute.WorkflowEventsCount(request.events.length), + ...Attribute.WorkflowExecutionMode(mode), + }); + + let session: WorkflowSession; + let execution: Promise; + switch (request.type) { + case 'replay': { + const started = await createWorkflowSession(request); + session = started.session; + execution = started.execution; + break; + } + case 'resume': { + session = request.session; + const resumed = session.resume(request.events); + if (resumed.type === 'replay') return resumed; + execution = resumed.execution; + break; + } + } + + span?.setAttributes({ + ...Attribute.WorkflowArgumentsCount(session.argumentCount), + }); + + try { + const completed = await execution; + span?.setAttributes({ + ...Attribute.WorkflowResultType(completed.resultType), + }); + return { type: 'completed', output: completed.output }; + } catch (error) { + if (WorkflowSuspension.is(error)) { + if (span) applyWorkflowSuspensionToSpan(error, span); + return { type: 'suspended', suspension: error, session }; + } + throw error; + } + }); +} + export async function runWorkflow( workflowCode: string, workflowRun: WorkflowRun, @@ -140,8 +280,7 @@ export async function runWorkflow( encryptionKey: CryptoKey | undefined, /** * Optional per-run cache for replay payload preparation and immutable final - * values. Owned by the inline replay loop so it survives fresh VM contexts - * created by successive iterations of this invocation. + * values. Owned by the inline execution loop for this invocation. */ replayPayloadCache: ReplayPayloadCache = new ReplayPayloadCache( encryptionKey @@ -154,14 +293,36 @@ export async function runWorkflow( */ runReadyBarrier?: Promise ): Promise { - return trace(`workflow.run ${workflowRun.workflowName}`, async (span) => { - span?.setAttributes({ - ...Attribute.WorkflowName(workflowRun.workflowName), - ...Attribute.WorkflowRunId(workflowRun.runId), - ...Attribute.WorkflowRunStatus(workflowRun.status), - ...Attribute.WorkflowEventsCount(events.length), - }); + const result = await executeWorkflow({ + type: 'replay', + workflowCode, + workflowRun, + events, + encryptionKey, + replayPayloadCache, + runReadyBarrier, + }); + if (result.type === 'replay') { + throw new WorkflowRuntimeError( + `Fresh workflow "${workflowRun.runId}" unexpectedly requested replay` + ); + } + if (result.type === 'suspended') throw result.suspension; + return result.output; +} +function createWorkflowSession({ + workflowCode, + workflowRun, + events, + encryptionKey, + replayPayloadCache, + runReadyBarrier, +}: WorkflowSessionOptions): Promise<{ + session: WorkflowSession; + execution: Promise; +}> { + return (async () => { const startedAt = workflowRun.startedAt; if (!startedAt) { throw new WorkflowRuntimeError( @@ -209,7 +370,32 @@ export async function runWorkflow( fixedTimestamp, }); - const workflowDiscontinuation = withResolvers(); + const initialInterruption = withResolvers(); + let state: WorkflowSessionState = { + type: 'running', + interruption: initialInterruption, + }; + + const onWorkflowError = (error: Error): void => { + switch (state.type) { + case 'running': { + const { interruption } = state; + state = WorkflowSuspension.is(error) + ? { type: 'suspended', suspension: error } + : { type: 'failed', error }; + interruption.reject(error); + return; + } + case 'suspended': + state = updateSuspendedSession(state.suspension, error); + return; + case 'failed': + case 'replay': + case 'completed': + return; + } + state satisfies never; + }; const ulid = monotonicFactory(() => vmGlobalThis.Math.random()); const generateNanoid = nanoid.customRandom(nanoid.urlAlphabet, 21, (size) => @@ -226,7 +412,7 @@ export async function runWorkflow( updateTimestamp(+event.createdAt); }, onUnconsumedEvent: (event) => { - workflowDiscontinuation.reject( + onWorkflowError( new ReplayDivergenceError( `Replay could not consume event: eventType=${event.eventType}, correlationId=${event.correlationId}, eventId=${event.eventId}.`, { eventId: event.eventId } @@ -240,7 +426,7 @@ export async function runWorkflow( runId: workflowRun.runId, encryptionKey, globalThis: vmGlobalThis, - onWorkflowError: workflowDiscontinuation.reject, + onWorkflowError, eventsConsumer, // Correlation IDs (step_/wait_/hook_) are derived from `generateUlid`, so // the time prefix fed to `ulid()` MUST be replay-stable across every @@ -873,48 +1059,16 @@ export async function runWorkflow( ); await workflowContext.promiseQueue; - span?.setAttributes({ - ...Attribute.WorkflowArgumentsCount(args.length), - }); - - // Invoke user workflow - try { - const result = await Promise.race([ - workflowFn(...args), - workflowDiscontinuation.promise, - ]); - - const dehydrated = await dehydrateWorkflowReturnValue( - result, - workflowRun.runId, - encryptionKey, - vmGlobalThis, - false, - // Gate payload compression on the run's specVersion: only runs - // marked as possibly containing compressed payloads (spec >= 5) - // get gzip data. - (workflowRun.specVersion ?? 0) >= SPEC_VERSION_SUPPORTS_COMPRESSION - ); + const workflowExecution = (async (): Promise => { + return await workflowFn(...args); + })(); - span?.setAttributes({ - ...Attribute.WorkflowResultType(typeof result), - }); - - await drainPendingQueueItems( - workflowRun.runId, - workflowContext.invocationsQueue, - vmGlobalThis, - workflowRun, - 'completed', - runReadyBarrier - ); - - return dehydrated; - } catch (err) { + const failWorkflow = async (error: unknown): Promise => { + state = { type: 'completed' }; // Control-flow signals are handled by the runtime and do not mean the // workflow has terminally failed. - if (WorkflowSuspension.is(err) || ReplayDivergenceError.is(err)) { - throw err; + if (WorkflowSuspension.is(error) || ReplayDivergenceError.is(error)) { + throw error; } await drainPendingQueueItems( @@ -926,7 +1080,97 @@ export async function runWorkflow( runReadyBarrier ); - throw err; - } - }); + throw error; + }; + + const completeWorkflow = async ( + result: unknown + ): Promise => { + state = { type: 'completed' }; + try { + const output = await dehydrateWorkflowReturnValue( + result, + workflowRun.runId, + encryptionKey, + vmGlobalThis, + false, + // Gate payload compression on the run's specVersion: only runs + // marked as possibly containing compressed payloads (spec >= 5) + // get gzip data. + (workflowRun.specVersion ?? 0) >= SPEC_VERSION_SUPPORTS_COMPRESSION + ); + + await drainPendingQueueItems( + workflowRun.runId, + workflowContext.invocationsQueue, + vmGlobalThis, + workflowRun, + 'completed', + runReadyBarrier + ); + + return { output, resultType: typeof result }; + } catch (error) { + return failWorkflow(error); + } + }; + + const waitForExecution = async ( + interruption: PromiseWithResolvers + ): Promise => { + let result: unknown; + try { + result = await Promise.race([workflowExecution, interruption.promise]); + } catch (error) { + if (state.type === 'suspended' && error === state.suspension) { + throw error; + } + return failWorkflow(error); + } + return completeWorkflow(result); + }; + + const session: WorkflowSession = { + workflowRun, + argumentCount: args.length, + resume(nextEvents) { + switch (state.type) { + case 'suspended': { + const consumedEvents = eventsConsumer.events; + const isStrictExtension = + nextEvents.length > consumedEvents.length && + consumedEvents.every( + (event, index) => event.eventId === nextEvents[index]?.eventId + ); + if (!isStrictExtension) { + state = { type: 'replay' }; + return { type: 'replay' }; + } + const interruption = withResolvers(); + state = { type: 'running', interruption }; + eventsConsumer.append(nextEvents.slice(consumedEvents.length)); + return { + type: 'resumed', + execution: waitForExecution(interruption), + }; + } + case 'failed': + return { type: 'resumed', execution: failWorkflow(state.error) }; + case 'replay': + return { type: 'replay' }; + case 'completed': + case 'running': + throw new WorkflowRuntimeError( + `Cannot resume ${state.type} workflow "${workflowRun.runId}"` + ); + } + state satisfies never; + }, + }; + + return { + session, + execution: waitForExecution(initialInterruption), + }; + })(); } From bab996c703e93e51675755f664aa6588a24444e7 Mon Sep 17 00:00:00 2001 From: Nathan Colosimo <110621881+NathanColosimo@users.noreply.github.com> Date: Fri, 17 Jul 2026 11:45:06 -0700 Subject: [PATCH 02/43] refactor(core): simplify retained-session control flow - executeWorkflow overloads: a fresh replay request can no longer return { type: 'replay' }, deleting the runtime invariant throw and runWorkflow's dead branch - isSameSuspensionBoundary reduced to the steps-array comparison (all suspension counts are derived from steps in the constructor) - runtime loop initializes workflowResult with a ternary --- packages/core/src/runtime.ts | 24 ++++++++---------------- packages/core/src/workflow.ts | 28 +++++++++++++++------------- 2 files changed, 23 insertions(+), 29 deletions(-) diff --git a/packages/core/src/runtime.ts b/packages/core/src/runtime.ts index a085c44884..a5b6c52d41 100644 --- a/packages/core/src/runtime.ts +++ b/packages/core/src/runtime.ts @@ -1460,16 +1460,14 @@ export function workflowEntrypoint( executionMode, }); replayStart = Date.now(); - let workflowResult: WorkflowExecutionResult = { - type: 'replay', - }; - if (workflowExecution.type === 'retained') { - workflowResult = await executeWorkflow({ - type: 'resume', - session: workflowExecution.session, - events, - }); - } + let workflowResult: WorkflowExecutionResult = + workflowExecution.type === 'retained' + ? await executeWorkflow({ + type: 'resume', + session: workflowExecution.session, + events, + }) + : { type: 'replay' }; if (workflowResult.type === 'replay') { executionMode = 'replay'; @@ -1510,12 +1508,6 @@ export function workflowEntrypoint( throw workflowResult.suspension; } - if (workflowResult.type === 'replay') { - throw new Error( - 'Invariant violation: fresh workflow execution requested another replay' - ); - } - const result = workflowResult.output; runtimeLogger.debug('Workflow execution completed', { workflowRunId: runId, diff --git a/packages/core/src/workflow.ts b/packages/core/src/workflow.ts index 1fe6668470..b7517bc3a9 100644 --- a/packages/core/src/workflow.ts +++ b/packages/core/src/workflow.ts @@ -152,17 +152,13 @@ type WorkflowSessionState = | { readonly type: 'replay' } | { readonly type: 'completed' }; +// The suspension counts are all derived from `steps` in the WorkflowSuspension +// constructor, so identical items mean an identical boundary. function isSameSuspensionBoundary( previous: WorkflowSuspension, next: WorkflowSuspension ): boolean { return ( - previous.stepCount === next.stepCount && - previous.hookCount === next.hookCount && - previous.waitCount === next.waitCount && - previous.attributeCount === next.attributeCount && - previous.hookDisposedCount === next.hookDisposedCount && - previous.abortCount === next.abortCount && previous.steps.length === next.steps.length && previous.steps.every((item, index) => item === next.steps[index]) ); @@ -208,8 +204,7 @@ type WorkflowExecutionRequest = readonly events: Event[]; }; -export type WorkflowExecutionResult = - | { readonly type: 'replay' } +type WorkflowReplayResult = | { readonly type: 'completed'; readonly output: unknown } | { readonly type: 'suspended'; @@ -217,6 +212,18 @@ export type WorkflowExecutionResult = readonly session: WorkflowSession; }; +export type WorkflowExecutionResult = + | { readonly type: 'replay' } + | WorkflowReplayResult; + +// A fresh replay always completes or suspends; only resuming a session can +// request another replay (stale session or diverged event prefix). +export async function executeWorkflow( + request: { readonly type: 'replay' } & WorkflowSessionOptions +): Promise; +export async function executeWorkflow( + request: WorkflowExecutionRequest +): Promise; export async function executeWorkflow( request: WorkflowExecutionRequest ): Promise { @@ -302,11 +309,6 @@ export async function runWorkflow( replayPayloadCache, runReadyBarrier, }); - if (result.type === 'replay') { - throw new WorkflowRuntimeError( - `Fresh workflow "${workflowRun.runId}" unexpectedly requested replay` - ); - } if (result.type === 'suspended') throw result.suspension; return result.output; } From bc5a318b70af4bc2c3570ce8058da9abedec3a5f Mon Sep 17 00:00:00 2001 From: Nathan Colosimo <110621881+NathanColosimo@users.noreply.github.com> Date: Fri, 17 Jul 2026 12:13:50 -0700 Subject: [PATCH 03/43] fix(core): decline retention for VMs that ran host-timed async work crypto.subtle.digest is the only sandbox API whose promise resolves on host timing rather than from the event log, so a workflow racing it against a step can advance while suspended and diverge from what replay reconstructs. A sticky usedHostAsync bit on the VM context makes canRetainWorkflowSession fall back to ordinary replay for such VMs; a quiescent step-only VM remains a pure function of the consumed event prefix and stays retainable. --- packages/core/src/retained-vm-loop.test.ts | 29 +++++++++++++++++++--- packages/core/src/runtime.ts | 7 +++++- packages/core/src/vm/index.ts | 13 +++++++++- packages/core/src/workflow.ts | 8 ++++++ 4 files changed, 51 insertions(+), 6 deletions(-) diff --git a/packages/core/src/retained-vm-loop.test.ts b/packages/core/src/retained-vm-loop.test.ts index d30740ba69..6f54e0fcd9 100644 --- a/packages/core/src/retained-vm-loop.test.ts +++ b/packages/core/src/retained-vm-loop.test.ts @@ -40,13 +40,25 @@ const twoStepWorkflow = `const s1 = globalThis[Symbol.for("WORKFLOW_USE_STEP")]( } globalThis.__private_workflows = new Map([["workflow", workflow]]);`; +// `crypto.subtle.digest` resolves on host timing, not from the event log, so +// this VM can advance while suspended — the loop must decline retention. +const hostAsyncWorkflow = `const s1 = globalThis[Symbol.for("WORKFLOW_USE_STEP")]("r_s1"); + const s2 = globalThis[Symbol.for("WORKFLOW_USE_STEP")]("r_s2"); + async function workflow() { + await crypto.subtle.digest("SHA-256", new Uint8Array(8)); + const a = await s1(); + const b = await s2(); + return a + b; + } + globalThis.__private_workflows = new Map([["workflow", workflow]]);`; + registerStepFunction('r_s1', async () => 10); registerStepFunction('r_s2', async () => 20); // Drive the full workflow handler over a stateful (dynamic) event log so the // inline loop makes real progress across its own writes, exactly like a World. // Non-turbo (no runInput, attempt 2) to keep the path simple and deterministic. -async function drive(runId: string) { +async function drive(runId: string, workflowCode = twoStepWorkflow) { const run: WorkflowRun = { runId, workflowName: 'workflow', @@ -126,9 +138,7 @@ async function drive(runId: string) { getEncryptionKeyForRun: vi.fn(async () => undefined), } as any); - await workflowEntrypoint(twoStepWorkflow)( - new Request('https://example.test') - ); + await workflowEntrypoint(workflowCode)(new Request('https://example.test')); const runCompleted = createdEvents.find( (e) => e.eventType === 'run_completed' @@ -175,4 +185,15 @@ describe('retained VM through the inline replay loop', () => { // And it produces the identical dehydrated result as the replay path. expect(on.output).toEqual(off.output); }); + + it('declines retention for a VM that ran host-timed async work', async () => { + const { vmBuilds, output } = await drive( + 'wrun_retained_host_async', + hostAsyncWorkflow + ); + expect(output).toBeInstanceOf(Uint8Array); + // The digest marks the VM as non-quiescent, so every iteration replays + // in a fresh VM even though retention is on by default. + expect(vmBuilds).toBeGreaterThan(1); + }); }); diff --git a/packages/core/src/runtime.ts b/packages/core/src/runtime.ts index a5b6c52d41..1e629c5292 100644 --- a/packages/core/src/runtime.ts +++ b/packages/core/src/runtime.ts @@ -318,11 +318,14 @@ function openHookAndWaitState(events: Event[]): { /** * Retain only pure step boundaries with no out-of-band continuation source. - * Attributes require replay; hooks and waits can wake another invocation. + * Attributes require replay; hooks and waits can wake another invocation; a + * VM that ran host-timed async work can advance while suspended, so its live + * state may not match what a replay of the event log reconstructs. * `WORKFLOW_RETAINED_VM=0` disables retention entirely. */ function canRetainWorkflowSession( suspension: WorkflowSuspension, + session: WorkflowSession, events: Event[] ): boolean { return ( @@ -333,6 +336,7 @@ function canRetainWorkflowSession( suspension.attributeCount === 0 && suspension.hookDisposedCount === 0 && suspension.abortCount === 0 && + !session.usedHostAsync() && !hasOpenHookOrWait(events) ); } @@ -1498,6 +1502,7 @@ export function workflowEntrypoint( if (workflowResult.type === 'suspended') { workflowExecution = canRetainWorkflowSession( workflowResult.suspension, + workflowResult.session, events ) ? { diff --git a/packages/core/src/vm/index.ts b/packages/core/src/vm/index.ts index c0003bdc39..d0b868e1df 100644 --- a/packages/core/src/vm/index.ts +++ b/packages/core/src/vm/index.ts @@ -58,7 +58,17 @@ export function createContext(options: CreateContextOptions) { const randomUUID = createRandomUUID(rng); + // `crypto.subtle.digest` is the only API in the sandbox whose promise + // resolves on host timing rather than from the event log, so it is the only + // way a suspended workflow can make progress the event log cannot replay. + // Record its use so the runtime falls back to ordinary replay (see + // `canRetainWorkflowSession`). + let usedHostAsync = false; const boundDigest = originalSubtle.digest.bind(originalSubtle); + const digest: typeof boundDigest = (...args) => { + usedHostAsync = true; + return boundDigest(...args); + }; g.crypto = new Proxy(originalCrypto, { get(target, prop) { @@ -78,7 +88,7 @@ export function createContext(options: CreateContextOptions) { ); }; } else if (prop === 'digest') { - return boundDigest; + return digest; } return target[prop as keyof typeof originalSubtle]; }, @@ -122,5 +132,6 @@ export function createContext(options: CreateContextOptions) { updateTimestamp: (timestamp: number) => { fixedTimestamp = timestamp; }, + usedHostAsync: () => usedHostAsync, }; } diff --git a/packages/core/src/workflow.ts b/packages/core/src/workflow.ts index b7517bc3a9..6989647953 100644 --- a/packages/core/src/workflow.ts +++ b/packages/core/src/workflow.ts @@ -177,6 +177,12 @@ function updateSuspendedSession( export interface WorkflowSession { readonly workflowRun: WorkflowRun; readonly argumentCount: number; + /** + * Whether the VM ran host-timed async work (`crypto.subtle.digest`). Such a + * VM can advance while suspended, so its live state may diverge from what a + * replay of the durable event log reconstructs — it must not be retained. + */ + usedHostAsync(): boolean; resume(events: Event[]): WorkflowSessionResumeResult; } @@ -367,6 +373,7 @@ function createWorkflowSession({ context, globalThis: vmGlobalThis, updateTimestamp, + usedHostAsync, } = createContext({ seed: `${workflowRun.runId}:${workflowRun.workflowName}:${workflowRun.deploymentId}`, fixedTimestamp, @@ -1135,6 +1142,7 @@ function createWorkflowSession({ const session: WorkflowSession = { workflowRun, argumentCount: args.length, + usedHostAsync, resume(nextEvents) { switch (state.type) { case 'suspended': { From 106936a0686d793738fc1ca6febf0c105b67df46 Mon Sep 17 00:00:00 2001 From: Nathan Colosimo <110621881+NathanColosimo@users.noreply.github.com> Date: Fri, 17 Jul 2026 12:27:24 -0700 Subject: [PATCH 04/43] fix(core): track all host-timed async VM APIs for retention Atomics.waitAsync (a wall-clock timer via SharedArrayBuffer) and the async WebAssembly compilation entry points resolve on host timing just like crypto.subtle.digest. Wrap every such intrinsic in createContext so usedHostAsync covers the complete set; dynamic import() settles within a microtask and cannot advance a suspended VM. --- packages/core/src/vm/index.test.ts | 53 ++++++++++++++++++++++++++++++ packages/core/src/vm/index.ts | 31 ++++++++++++++--- packages/core/src/workflow.ts | 7 ++-- 3 files changed, 83 insertions(+), 8 deletions(-) diff --git a/packages/core/src/vm/index.test.ts b/packages/core/src/vm/index.test.ts index 0381b60f63..1fad27b36e 100644 --- a/packages/core/src/vm/index.test.ts +++ b/packages/core/src/vm/index.test.ts @@ -257,3 +257,56 @@ describe('createContext', () => { expect(result).toBe('undefined'); }); }); + +describe('usedHostAsync', () => { + it('starts false and ignores log-driven and deterministic APIs', () => { + const { context, usedHostAsync } = createContext({ seed, fixedTimestamp }); + + vm.runInContext( + 'Math.random(); Date.now(); crypto.randomUUID(); crypto.getRandomValues(new Uint8Array(4))', + context + ); + expect(usedHostAsync()).toBe(false); + }); + + it('flips on crypto.subtle.digest', async () => { + const { context, usedHostAsync } = createContext({ seed, fixedTimestamp }); + + await vm.runInContext( + 'crypto.subtle.digest("SHA-256", new Uint8Array(1))', + context + ); + expect(usedHostAsync()).toBe(true); + }); + + it('flips on Atomics.waitAsync', () => { + const { context, usedHostAsync } = createContext({ seed, fixedTimestamp }); + + vm.runInContext( + 'Atomics.waitAsync(new Int32Array(new SharedArrayBuffer(4)), 0, 1)', + context + ); + expect(usedHostAsync()).toBe(true); + }); + + it('flips on WebAssembly.compile', async () => { + const { context, usedHostAsync } = createContext({ seed, fixedTimestamp }); + + // Minimal valid empty module: magic + version. + await vm.runInContext( + 'WebAssembly.compile(new Uint8Array([0,97,115,109,1,0,0,0]))', + context + ); + expect(usedHostAsync()).toBe(true); + }); + + it('flips on WebAssembly.instantiate', async () => { + const { context, usedHostAsync } = createContext({ seed, fixedTimestamp }); + + await vm.runInContext( + 'WebAssembly.instantiate(new Uint8Array([0,97,115,109,1,0,0,0]))', + context + ); + expect(usedHostAsync()).toBe(true); + }); +}); diff --git a/packages/core/src/vm/index.ts b/packages/core/src/vm/index.ts index d0b868e1df..af61842895 100644 --- a/packages/core/src/vm/index.ts +++ b/packages/core/src/vm/index.ts @@ -58,12 +58,33 @@ export function createContext(options: CreateContextOptions) { const randomUUID = createRandomUUID(rng); - // `crypto.subtle.digest` is the only API in the sandbox whose promise - // resolves on host timing rather than from the event log, so it is the only - // way a suspended workflow can make progress the event log cannot replay. - // Record its use so the runtime falls back to ordinary replay (see - // `canRetainWorkflowSession`). + // Track every sandbox API whose promise resolves on host timing rather + // than from the event log: `crypto.subtle.digest`, `Atomics.waitAsync` + // (a wall-clock timer via SharedArrayBuffer), and the async `WebAssembly` + // compilation entry points. These are the only ways a suspended workflow + // can make progress the event log cannot replay, so the runtime declines + // to retain a VM that used any of them (see `canRetainWorkflowSession`). + // Dynamic `import()` settles within a microtask (rejected: no + // `importModuleDynamically`), so it cannot advance a suspended VM. let usedHostAsync = false; + const trackHostAsync = ( + target: Record, + method: string + ): void => { + const original = target[method]; + if (typeof original !== 'function') return; + target[method] = (...args: unknown[]) => { + usedHostAsync = true; + return Reflect.apply(original, target, args); + }; + }; + const intrinsics = g as unknown as Record>; + trackHostAsync(intrinsics.Atomics, 'waitAsync'); + trackHostAsync(intrinsics.WebAssembly, 'compile'); + trackHostAsync(intrinsics.WebAssembly, 'instantiate'); + trackHostAsync(intrinsics.WebAssembly, 'compileStreaming'); + trackHostAsync(intrinsics.WebAssembly, 'instantiateStreaming'); + const boundDigest = originalSubtle.digest.bind(originalSubtle); const digest: typeof boundDigest = (...args) => { usedHostAsync = true; diff --git a/packages/core/src/workflow.ts b/packages/core/src/workflow.ts index 6989647953..152cd0a6a5 100644 --- a/packages/core/src/workflow.ts +++ b/packages/core/src/workflow.ts @@ -178,9 +178,10 @@ export interface WorkflowSession { readonly workflowRun: WorkflowRun; readonly argumentCount: number; /** - * Whether the VM ran host-timed async work (`crypto.subtle.digest`). Such a - * VM can advance while suspended, so its live state may diverge from what a - * replay of the durable event log reconstructs — it must not be retained. + * Whether the VM ran host-timed async work (`crypto.subtle.digest`, + * `Atomics.waitAsync`, async `WebAssembly` compilation). Such a VM can + * advance while suspended, so its live state may diverge from what a replay + * of the durable event log reconstructs — it must not be retained. */ usedHostAsync(): boolean; resume(events: Event[]): WorkflowSessionResumeResult; From 9001a31da7c26166b519908eb17ec0e3e355055d Mon Sep 17 00:00:00 2001 From: Nathan Colosimo <110621881+NathanColosimo@users.noreply.github.com> Date: Fri, 17 Jul 2026 12:33:55 -0700 Subject: [PATCH 05/43] feat(core): compute crypto.subtle.digest synchronously in the sandbox node:crypto createHash produces byte-identical values to WebCrypto and settles the digest promise on a deterministic microtask instead of host threadpool timing. A digest can therefore never advance a suspended workflow, so digest-using VMs stay retainable; only Atomics.waitAsync and async WebAssembly compilation remain host-timed. createHash is stable and undeprecated on Node 18-26 (DEP0179 only removed the direct Hash constructor). --- .changeset/retain-workflow-vm.md | 2 +- .../v5/api-reference/workflow-globals.mdx | 2 +- .../docs/v5/configuration/runtime-tuning.mdx | 2 +- packages/core/src/retained-vm-loop.test.ts | 27 +++++++- packages/core/src/vm/index.test.ts | 62 ++++++++++++++++++- packages/core/src/vm/index.ts | 55 ++++++++++++---- packages/core/src/workflow.test.ts | 26 ++++---- packages/core/src/workflow.ts | 8 +-- 8 files changed, 148 insertions(+), 36 deletions(-) diff --git a/.changeset/retain-workflow-vm.md b/.changeset/retain-workflow-vm.md index 37b874325e..d6b73c8ab6 100644 --- a/.changeset/retain-workflow-vm.md +++ b/.changeset/retain-workflow-vm.md @@ -3,4 +3,4 @@ 'workflow': patch --- -Retain workflow execution across inline steps within one invocation. `WORKFLOW_RETAINED_VM=0` disables retention. +Retain workflow execution across inline steps within one invocation; `WORKFLOW_RETAINED_VM=0` disables retention. `crypto.subtle.digest` in workflow functions now computes synchronously via `node:crypto`, so digest timing is deterministic under replay. diff --git a/docs/content/docs/v5/api-reference/workflow-globals.mdx b/docs/content/docs/v5/api-reference/workflow-globals.mdx index 6ec32b6b0b..a9824c91d6 100644 --- a/docs/content/docs/v5/api-reference/workflow-globals.mdx +++ b/docs/content/docs/v5/api-reference/workflow-globals.mdx @@ -26,7 +26,7 @@ These APIs are available but are **seeded or fixed** to ensure deterministic beh | [`Date`](https://developer.mozilla.org/en-US/docs/Web/JavaScript/Reference/Global_Objects/Date) / `Date.now()` / `new Date()` | Returns a fixed timestamp that advances with the workflow's logical clock | | [`crypto.getRandomValues()`](https://developer.mozilla.org/en-US/docs/Web/API/Crypto/getRandomValues) | Seeded — produces deterministic output for a given workflow run | | [`crypto.randomUUID()`](https://developer.mozilla.org/en-US/docs/Web/API/Crypto/randomUUID) | Seeded — produces deterministic UUIDs for a given workflow run | -| [`crypto.subtle.digest()`](https://developer.mozilla.org/en-US/docs/Web/API/SubtleCrypto/digest) | Passes through to the real implementation (SHA-256, etc. are deterministic by nature) | +| [`crypto.subtle.digest()`](https://developer.mozilla.org/en-US/docs/Web/API/SubtleCrypto/digest) | Computed synchronously via `node:crypto` (values are byte-identical to WebCrypto), so the promise settles at a deterministic point during replay | You can safely use `Math.random()`, `Date.now()`, and `crypto.randomUUID()` in workflow functions. The framework ensures these return the same values across replays. diff --git a/docs/content/docs/v5/configuration/runtime-tuning.mdx b/docs/content/docs/v5/configuration/runtime-tuning.mdx index f2f1a98e53..e5b65c5eaa 100644 --- a/docs/content/docs/v5/configuration/runtime-tuning.mdx +++ b/docs/content/docs/v5/configuration/runtime-tuning.mdx @@ -79,7 +79,7 @@ For example, a workflow can run a 10-minute inline step even with `WORKFLOW_REPL - Default: enabled - Keeps the suspended workflow VM alive across inline steps within one invocation, so each iteration of the inline loop appends only the newly written events instead of replaying the whole event log in a fresh VM. -- Suspensions involving hooks, waits, or attributes — and any replay divergence — always fall back to a full replay. +- Suspensions involving hooks, waits, or attributes, workflows that used host-timed async APIs (`Atomics.waitAsync`, async `WebAssembly` compilation), and any replay divergence always fall back to a full replay. - Set `0` or `false` to replay from scratch in a fresh VM on every iteration. ### `WORKFLOW_INLINE_OWNERSHIP` diff --git a/packages/core/src/retained-vm-loop.test.ts b/packages/core/src/retained-vm-loop.test.ts index 6f54e0fcd9..ea94abd6ea 100644 --- a/packages/core/src/retained-vm-loop.test.ts +++ b/packages/core/src/retained-vm-loop.test.ts @@ -40,9 +40,21 @@ const twoStepWorkflow = `const s1 = globalThis[Symbol.for("WORKFLOW_USE_STEP")]( } globalThis.__private_workflows = new Map([["workflow", workflow]]);`; -// `crypto.subtle.digest` resolves on host timing, not from the event log, so +// `Atomics.waitAsync` resolves on host timing, not from the event log, so // this VM can advance while suspended — the loop must decline retention. const hostAsyncWorkflow = `const s1 = globalThis[Symbol.for("WORKFLOW_USE_STEP")]("r_s1"); + const s2 = globalThis[Symbol.for("WORKFLOW_USE_STEP")]("r_s2"); + async function workflow() { + await Atomics.waitAsync(new Int32Array(new SharedArrayBuffer(4)), 0, 1).value; + const a = await s1(); + const b = await s2(); + return a + b; + } + globalThis.__private_workflows = new Map([["workflow", workflow]]);`; + +// `crypto.subtle.digest` computes synchronously via node:crypto, so a +// digest-using VM stays quiescent at suspension and remains retainable. +const digestWorkflow = `const s1 = globalThis[Symbol.for("WORKFLOW_USE_STEP")]("r_s1"); const s2 = globalThis[Symbol.for("WORKFLOW_USE_STEP")]("r_s2"); async function workflow() { await crypto.subtle.digest("SHA-256", new Uint8Array(8)); @@ -192,8 +204,17 @@ describe('retained VM through the inline replay loop', () => { hostAsyncWorkflow ); expect(output).toBeInstanceOf(Uint8Array); - // The digest marks the VM as non-quiescent, so every iteration replays - // in a fresh VM even though retention is on by default. + // Atomics.waitAsync marks the VM as non-quiescent, so every iteration + // replays in a fresh VM even though retention is on by default. expect(vmBuilds).toBeGreaterThan(1); }); + + it('retains a VM that used the synchronous crypto.subtle.digest', async () => { + const { vmBuilds, output } = await drive( + 'wrun_retained_digest', + digestWorkflow + ); + expect(output).toBeInstanceOf(Uint8Array); + expect(vmBuilds).toBe(1); + }); }); diff --git a/packages/core/src/vm/index.test.ts b/packages/core/src/vm/index.test.ts index 1fad27b36e..dfb42e6089 100644 --- a/packages/core/src/vm/index.test.ts +++ b/packages/core/src/vm/index.test.ts @@ -269,14 +269,14 @@ describe('usedHostAsync', () => { expect(usedHostAsync()).toBe(false); }); - it('flips on crypto.subtle.digest', async () => { + it('does not flip on the synchronous crypto.subtle.digest', async () => { const { context, usedHostAsync } = createContext({ seed, fixedTimestamp }); await vm.runInContext( 'crypto.subtle.digest("SHA-256", new Uint8Array(1))', context ); - expect(usedHostAsync()).toBe(true); + expect(usedHostAsync()).toBe(false); }); it('flips on Atomics.waitAsync', () => { @@ -310,3 +310,61 @@ describe('usedHostAsync', () => { expect(usedHostAsync()).toBe(true); }); }); + +describe('crypto.subtle.digest', () => { + it.each([ + 'SHA-1', + 'SHA-256', + 'SHA-384', + 'SHA-512', + ])('matches WebCrypto for %s', async (algorithm) => { + const { context } = createContext({ seed, fixedTimestamp }); + + const result = await vm.runInContext( + `crypto.subtle.digest(${JSON.stringify(algorithm)}, new Uint8Array([1,2,3,4,5,255,0,128]))`, + context + ); + const expected = await globalThis.crypto.subtle.digest( + algorithm, + new Uint8Array([1, 2, 3, 4, 5, 255, 0, 128]) + ); + expect(result).toBeInstanceOf(ArrayBuffer); + expect(new Uint8Array(result)).toEqual(new Uint8Array(expected)); + }); + + it('accepts the { name } algorithm form and ArrayBuffer input', async () => { + const { context } = createContext({ seed, fixedTimestamp }); + + const result = await vm.runInContext( + 'crypto.subtle.digest({ name: "sha-256" }, new ArrayBuffer(4))', + context + ); + const expected = await globalThis.crypto.subtle.digest( + 'SHA-256', + new ArrayBuffer(4) + ); + expect(new Uint8Array(result)).toEqual(new Uint8Array(expected)); + }); + + it('respects typed-array subviews', async () => { + const { context } = createContext({ seed, fixedTimestamp }); + + const result = await vm.runInContext( + 'const bytes = new Uint8Array([9, 1, 2, 9]); crypto.subtle.digest("SHA-256", bytes.subarray(1, 3))', + context + ); + const expected = await globalThis.crypto.subtle.digest( + 'SHA-256', + new Uint8Array([1, 2]) + ); + expect(new Uint8Array(result)).toEqual(new Uint8Array(expected)); + }); + + it('rejects unsupported algorithms like WebCrypto does', async () => { + const { context } = createContext({ seed, fixedTimestamp }); + + await expect( + vm.runInContext('crypto.subtle.digest("MD5", new Uint8Array(1))', context) + ).rejects.toMatchObject({ name: 'NotSupportedError' }); + }); +}); diff --git a/packages/core/src/vm/index.ts b/packages/core/src/vm/index.ts index af61842895..f10fcff956 100644 --- a/packages/core/src/vm/index.ts +++ b/packages/core/src/vm/index.ts @@ -1,3 +1,4 @@ +import { createHash } from 'node:crypto'; import { runInContext, createContext as vmCreateContext } from 'node:vm'; import { WorkflowRuntimeError } from '@workflow/errors'; import seedrandom from 'seedrandom'; @@ -10,6 +11,14 @@ export interface CreateContextOptions { fixedTimestamp: number; } +// WebCrypto digest algorithm names → node:crypto (OpenSSL) names. +const DIGEST_ALGORITHMS: Record = { + 'SHA-1': 'sha1', + 'SHA-256': 'sha256', + 'SHA-384': 'sha384', + 'SHA-512': 'sha512', +}; + /** * Creates a Node.js `vm.Context` configured to be usable for * executing workflow logic in a deterministic environment. @@ -59,13 +68,13 @@ export function createContext(options: CreateContextOptions) { const randomUUID = createRandomUUID(rng); // Track every sandbox API whose promise resolves on host timing rather - // than from the event log: `crypto.subtle.digest`, `Atomics.waitAsync` - // (a wall-clock timer via SharedArrayBuffer), and the async `WebAssembly` - // compilation entry points. These are the only ways a suspended workflow - // can make progress the event log cannot replay, so the runtime declines - // to retain a VM that used any of them (see `canRetainWorkflowSession`). - // Dynamic `import()` settles within a microtask (rejected: no - // `importModuleDynamically`), so it cannot advance a suspended VM. + // than from the event log: `Atomics.waitAsync` (a wall-clock timer via + // SharedArrayBuffer) and the async `WebAssembly` compilation entry points. + // These are the only ways a suspended workflow can make progress the event + // log cannot replay, so the runtime declines to retain a VM that used any + // of them (see `canRetainWorkflowSession`). Dynamic `import()` settles + // within a microtask (rejected: no `importModuleDynamically`), so it + // cannot advance a suspended VM. let usedHostAsync = false; const trackHostAsync = ( target: Record, @@ -85,10 +94,34 @@ export function createContext(options: CreateContextOptions) { trackHostAsync(intrinsics.WebAssembly, 'compileStreaming'); trackHostAsync(intrinsics.WebAssembly, 'instantiateStreaming'); - const boundDigest = originalSubtle.digest.bind(originalSubtle); - const digest: typeof boundDigest = (...args) => { - usedHostAsync = true; - return boundDigest(...args); + // `crypto.subtle.digest` computes synchronously via node:crypto, so its + // promise settles on a deterministic microtask instead of host threadpool + // timing — a digest can never advance a suspended workflow, and + // digest-using VMs stay retainable. Values are byte-identical to WebCrypto. + const digest = ( + algorithm: string | { name: string }, + data: ArrayBuffer | ArrayBufferView + ): Promise => { + try { + const name = typeof algorithm === 'string' ? algorithm : algorithm.name; + const ossl = DIGEST_ALGORITHMS[name.toUpperCase()]; + if (!ossl) { + throw new DOMException( + `Unrecognized algorithm name: ${name}`, + 'NotSupportedError' + ); + } + const bytes = ArrayBuffer.isView(data) + ? new Uint8Array(data.buffer, data.byteOffset, data.byteLength) + : new Uint8Array(data); + const hash = createHash(ossl).update(bytes).digest(); + // Copy out: small Buffers share the internal pool allocation. + const out = new ArrayBuffer(hash.byteLength); + new Uint8Array(out).set(hash); + return Promise.resolve(out); + } catch (error) { + return Promise.reject(error); + } }; g.crypto = new Proxy(originalCrypto, { diff --git a/packages/core/src/workflow.test.ts b/packages/core/src/workflow.test.ts index 66cc295e27..2af9c6de2b 100644 --- a/packages/core/src/workflow.test.ts +++ b/packages/core/src/workflow.test.ts @@ -361,16 +361,16 @@ describe('runWorkflow', () => { }; const workflowCode = ` const step = globalThis[Symbol.for("WORKFLOW_USE_STEP")]("step"); - async function digestRepeatedly() { - const bytes = new Uint8Array(8 * 1024 * 1024); - for (let index = 0; index < 8; index++) { - await crypto.subtle.digest("SHA-256", bytes); + async function waitRepeatedly() { + const shared = new Int32Array(new SharedArrayBuffer(4)); + for (let index = 0; index < 20; index++) { + await Atomics.waitAsync(shared, 0, 0, 25).value; } } async function workflow() { - await Promise.race([step(), digestRepeatedly()]); - console.log("retained:digest-completed"); - return "digest completed"; + await Promise.race([step(), waitRepeatedly()]); + console.log("retained:background-completed"); + return "background completed"; } ${getWorkflowTransformCode('workflow')}`; const events: Event[] = [ @@ -479,7 +479,7 @@ describe('runWorkflow', () => { noEncryptionKey, ops ) - ).toBe('digest completed'); + ).toBe('background completed'); log.mockRestore(); }); @@ -503,14 +503,14 @@ describe('runWorkflow', () => { const workflowCode = ` const step = globalThis[Symbol.for("WORKFLOW_USE_STEP")]("step"); const setAttributes = globalThis[Symbol.for("WORKFLOW_SET_ATTRIBUTES")]; - async function digestRepeatedly() { - const bytes = new Uint8Array(8 * 1024 * 1024); - for (let index = 0; index < 8; index++) { - await crypto.subtle.digest("SHA-256", bytes); + async function waitRepeatedly() { + const shared = new Int32Array(new SharedArrayBuffer(4)); + for (let index = 0; index < 20; index++) { + await Atomics.waitAsync(shared, 0, 0, 25).value; } } async function workflow() { - await Promise.race([step(), digestRepeatedly()]); + await Promise.race([step(), waitRepeatedly()]); void setAttributes([{ key: "stale", value: true }]); console.log("retained:discarded-completed"); return "discarded"; diff --git a/packages/core/src/workflow.ts b/packages/core/src/workflow.ts index 152cd0a6a5..5ff5991c41 100644 --- a/packages/core/src/workflow.ts +++ b/packages/core/src/workflow.ts @@ -178,10 +178,10 @@ export interface WorkflowSession { readonly workflowRun: WorkflowRun; readonly argumentCount: number; /** - * Whether the VM ran host-timed async work (`crypto.subtle.digest`, - * `Atomics.waitAsync`, async `WebAssembly` compilation). Such a VM can - * advance while suspended, so its live state may diverge from what a replay - * of the durable event log reconstructs — it must not be retained. + * Whether the VM ran host-timed async work (`Atomics.waitAsync`, async + * `WebAssembly` compilation). Such a VM can advance while suspended, so its + * live state may diverge from what a replay of the durable event log + * reconstructs — it must not be retained. */ usedHostAsync(): boolean; resume(events: Event[]): WorkflowSessionResumeResult; From 2abf8b8e44ffefe3a14315dd0511c9e568035f92 Mon Sep 17 00:00:00 2001 From: Nathan Colosimo <110621881+NathanColosimo@users.noreply.github.com> Date: Fri, 17 Jul 2026 12:40:48 -0700 Subject: [PATCH 06/43] fix(core): remove WeakRef and FinalizationRegistry from the sandbox GC observation depends on host GC timing that neither replay nor a retained VM can reconstruct from the event log. WeakMap/WeakSet stay available (they do not expose GC state). --- .changeset/retain-workflow-vm.md | 2 +- .../docs/v5/api-reference/workflow-globals.mdx | 1 + packages/core/src/vm/index.test.ts | 14 ++++++++++++++ packages/core/src/vm/index.ts | 6 ++++++ 4 files changed, 22 insertions(+), 1 deletion(-) diff --git a/.changeset/retain-workflow-vm.md b/.changeset/retain-workflow-vm.md index d6b73c8ab6..61cb327ee9 100644 --- a/.changeset/retain-workflow-vm.md +++ b/.changeset/retain-workflow-vm.md @@ -3,4 +3,4 @@ 'workflow': patch --- -Retain workflow execution across inline steps within one invocation; `WORKFLOW_RETAINED_VM=0` disables retention. `crypto.subtle.digest` in workflow functions now computes synchronously via `node:crypto`, so digest timing is deterministic under replay. +Retain workflow execution across inline steps within one invocation; `WORKFLOW_RETAINED_VM=0` disables retention. `crypto.subtle.digest` in workflow functions now computes synchronously via `node:crypto`, so digest timing is deterministic under replay. `WeakRef` and `FinalizationRegistry` are no longer exposed in workflow functions (GC observation cannot be replayed). diff --git a/docs/content/docs/v5/api-reference/workflow-globals.mdx b/docs/content/docs/v5/api-reference/workflow-globals.mdx index a9824c91d6..4f7eedf1d9 100644 --- a/docs/content/docs/v5/api-reference/workflow-globals.mdx +++ b/docs/content/docs/v5/api-reference/workflow-globals.mdx @@ -100,3 +100,4 @@ The following are **not available** in workflow functions. Move this logic to [s - **Global `fetch`**: Use [`import { fetch } from "workflow"`](/docs/api-reference/workflow/fetch) instead. See [fetch-in-workflow](/docs/errors/fetch-in-workflow). - **Timers**: `setTimeout`, `setInterval`, `setImmediate`, and their `clear*` counterparts. Use [`sleep()`](/docs/api-reference/workflow/sleep) instead. See [timeout-in-workflow](/docs/errors/timeout-in-workflow). - **`Buffer`**: Node.js-specific API. Use `Uint8Array` with `toBase64()` / `fromBase64()` / `toHex()` / `fromHex()` for binary data encoding, or `atob()` / `btoa()` for string-based base64. +- **`WeakRef` and `FinalizationRegistry`**: garbage-collection timing is not deterministic, so observing it would make workflow code impossible to replay. (`WeakMap` and `WeakSet` remain available — they do not expose GC state.) diff --git a/packages/core/src/vm/index.test.ts b/packages/core/src/vm/index.test.ts index dfb42e6089..a29a2e146f 100644 --- a/packages/core/src/vm/index.test.ts +++ b/packages/core/src/vm/index.test.ts @@ -368,3 +368,17 @@ describe('crypto.subtle.digest', () => { ).rejects.toMatchObject({ name: 'NotSupportedError' }); }); }); + +describe('GC observation', () => { + it('does not expose WeakRef or FinalizationRegistry', () => { + const { context } = createContext({ seed, fixedTimestamp }); + + expect(vm.runInContext('typeof WeakRef', context)).toBe('undefined'); + expect(vm.runInContext('typeof FinalizationRegistry', context)).toBe( + 'undefined' + ); + // WeakMap/WeakSet do not expose GC state and stay available. + expect(vm.runInContext('typeof WeakMap', context)).toBe('function'); + expect(vm.runInContext('typeof WeakSet', context)).toBe('function'); + }); +}); diff --git a/packages/core/src/vm/index.ts b/packages/core/src/vm/index.ts index f10fcff956..fdd0d10fbf 100644 --- a/packages/core/src/vm/index.ts +++ b/packages/core/src/vm/index.ts @@ -94,6 +94,12 @@ export function createContext(options: CreateContextOptions) { trackHostAsync(intrinsics.WebAssembly, 'compileStreaming'); trackHostAsync(intrinsics.WebAssembly, 'instantiateStreaming'); + // GC observation (`WeakRef.deref()`, finalizer callbacks) depends on host + // GC timing that neither replay nor a retained VM can reconstruct from the + // event log, so the sandbox does not expose it at all. + delete intrinsics.WeakRef; + delete intrinsics.FinalizationRegistry; + // `crypto.subtle.digest` computes synchronously via node:crypto, so its // promise settles on a deterministic microtask instead of host threadpool // timing — a digest can never advance a suspended workflow, and From 4905577767e060b03c61521601db073ffeb3bc75 Mon Sep 17 00:00:00 2001 From: Nathan Colosimo <110621881+NathanColosimo@users.noreply.github.com> Date: Fri, 17 Jul 2026 12:56:49 -0700 Subject: [PATCH 07/43] fix(core): enforce the BufferSource contract in the sandbox digest Reject non-BufferSource digest input with TypeError like WebCrypto does, via the native ArrayBuffer.prototype.byteLength brand check (works across vm realms). Previously a plain number was treated as a Uint8Array length, turning a small input into a giant allocation. --- packages/core/src/vm/index.test.ts | 15 +++++++++++++++ packages/core/src/vm/index.ts | 23 +++++++++++++++++++---- 2 files changed, 34 insertions(+), 4 deletions(-) diff --git a/packages/core/src/vm/index.test.ts b/packages/core/src/vm/index.test.ts index a29a2e146f..b2d694c5cd 100644 --- a/packages/core/src/vm/index.test.ts +++ b/packages/core/src/vm/index.test.ts @@ -382,3 +382,18 @@ describe('GC observation', () => { expect(vm.runInContext('typeof WeakSet', context)).toBe('function'); }); }); + +describe('crypto.subtle.digest input validation', () => { + it.each([ + ['a number', '2000000000'], + ['a plain object', '({})'], + ['a string', '"data"'], + ['null', 'null'], + ])('rejects %s with TypeError', async (_label, expression) => { + const { context } = createContext({ seed, fixedTimestamp }); + + await expect( + vm.runInContext(`crypto.subtle.digest("SHA-256", ${expression})`, context) + ).rejects.toThrow(TypeError); + }); +}); diff --git a/packages/core/src/vm/index.ts b/packages/core/src/vm/index.ts index fdd0d10fbf..ba67168173 100644 --- a/packages/core/src/vm/index.ts +++ b/packages/core/src/vm/index.ts @@ -19,6 +19,24 @@ const DIGEST_ALGORITHMS: Record = { 'SHA-512': 'sha512', }; +// biome-ignore lint/style/noNonNullAssertion: byteLength always exists on ArrayBuffer.prototype +const arrayBufferByteLength = Object.getOwnPropertyDescriptor( + ArrayBuffer.prototype, + 'byteLength' +)!.get!; + +// WebCrypto BufferSource conversion: views pass through; anything else must +// be a real ArrayBuffer (the native byteLength getter is a brand check that +// works across vm realms) so that e.g. a plain number is rejected with +// TypeError instead of allocating a Uint8Array of that length. +function toDigestBytes(data: ArrayBuffer | ArrayBufferView): Uint8Array { + if (ArrayBuffer.isView(data)) { + return new Uint8Array(data.buffer, data.byteOffset, data.byteLength); + } + arrayBufferByteLength.call(data); + return new Uint8Array(data); +} + /** * Creates a Node.js `vm.Context` configured to be usable for * executing workflow logic in a deterministic environment. @@ -117,10 +135,7 @@ export function createContext(options: CreateContextOptions) { 'NotSupportedError' ); } - const bytes = ArrayBuffer.isView(data) - ? new Uint8Array(data.buffer, data.byteOffset, data.byteLength) - : new Uint8Array(data); - const hash = createHash(ossl).update(bytes).digest(); + const hash = createHash(ossl).update(toDigestBytes(data)).digest(); // Copy out: small Buffers share the internal pool allocation. const out = new ArrayBuffer(hash.byteLength); new Uint8Array(out).set(hash); From 73ee4ae9a064aaddbfecfe3ae83bfbe78ad0d29d Mon Sep 17 00:00:00 2001 From: Nathan Colosimo <110621881+NathanColosimo@users.noreply.github.com> Date: Fri, 17 Jul 2026 13:15:31 -0700 Subject: [PATCH 08/43] fix(core): demote retention when suspension serialization draws randomness handleSuspension dehydrates step arguments with the live VM, and that serialization can execute user code (getters, WORKFLOW_SERIALIZE hooks). Randomness drawn there would desync the retained VM's future correlation IDs from what a fresh replay regenerates. Count every draw from the seeded stream at its single source in createContext and fall back to ordinary replay if handleSuspension consumed any. --- .../workflow-serde/workflow-serialize.mdx | 1 + packages/core/src/retained-vm-loop.test.ts | 21 +++++++++++++++++++ packages/core/src/runtime.ts | 15 +++++++++++++ packages/core/src/vm/index.ts | 13 +++++++++++- packages/core/src/workflow.ts | 10 +++++++++ 5 files changed, 59 insertions(+), 1 deletion(-) diff --git a/docs/content/docs/v5/api-reference/workflow-serde/workflow-serialize.mdx b/docs/content/docs/v5/api-reference/workflow-serde/workflow-serialize.mdx index 85c5eb8acf..a5380c3188 100644 --- a/docs/content/docs/v5/api-reference/workflow-serde/workflow-serialize.mdx +++ b/docs/content/docs/v5/api-reference/workflow-serde/workflow-serialize.mdx @@ -69,6 +69,7 @@ This method runs inside the workflow context and is subject to the same constrai - No Node.js-specific APIs (like `fs`, `path`, `crypto`, etc.) - No non-deterministic operations (like `Math.random()` or `Date.now()`) - No external network calls +- No side effects on workflow state — the method may run outside deterministic replay, so mutations would not be reconstructed Keep this method simple and focused on extracting data from the instance. diff --git a/packages/core/src/retained-vm-loop.test.ts b/packages/core/src/retained-vm-loop.test.ts index ea94abd6ea..a76f58f8aa 100644 --- a/packages/core/src/retained-vm-loop.test.ts +++ b/packages/core/src/retained-vm-loop.test.ts @@ -52,6 +52,18 @@ const hostAsyncWorkflow = `const s1 = globalThis[Symbol.for("WORKFLOW_USE_STEP") } globalThis.__private_workflows = new Map([["workflow", workflow]]);`; +// Serializing this step's arguments executes the getter, which draws from +// the VM's seeded random stream after the suspension — the loop must demote +// the session so future correlation IDs stay replayable. +const impureArgsWorkflow = `const s1 = globalThis[Symbol.for("WORKFLOW_USE_STEP")]("r_s1"); + const s2 = globalThis[Symbol.for("WORKFLOW_USE_STEP")]("r_s2"); + async function workflow() { + const a = await s1({ get x() { Math.random(); return 1; } }); + const b = await s2(); + return a + b; + } + globalThis.__private_workflows = new Map([["workflow", workflow]]);`; + // `crypto.subtle.digest` computes synchronously via node:crypto, so a // digest-using VM stays quiescent at suspension and remains retainable. const digestWorkflow = `const s1 = globalThis[Symbol.for("WORKFLOW_USE_STEP")]("r_s1"); @@ -209,6 +221,15 @@ describe('retained VM through the inline replay loop', () => { expect(vmBuilds).toBeGreaterThan(1); }); + it('demotes retention when argument serialization draws randomness', async () => { + const { vmBuilds, output } = await drive( + 'wrun_retained_impure_args', + impureArgsWorkflow + ); + expect(output).toBeInstanceOf(Uint8Array); + expect(vmBuilds).toBeGreaterThan(1); + }); + it('retains a VM that used the synchronous crypto.subtle.digest', async () => { const { vmBuilds, output } = await drive( 'wrun_retained_digest', diff --git a/packages/core/src/runtime.ts b/packages/core/src/runtime.ts index 1e629c5292..bd29c8e93e 100644 --- a/packages/core/src/runtime.ts +++ b/packages/core/src/runtime.ts @@ -1140,6 +1140,7 @@ export function workflowEntrypoint( | { readonly type: 'retained'; readonly session: WorkflowSession; + readonly drawCountAtSuspension: number; } = { type: 'replay' }; // Main replay loop @@ -1508,6 +1509,8 @@ export function workflowEntrypoint( ? { type: 'retained', session: workflowResult.session, + drawCountAtSuspension: + workflowResult.session.randomDrawCount(), } : { type: 'replay' }; throw workflowResult.suspension; @@ -1644,6 +1647,18 @@ export function workflowEntrypoint( eventLog: suspensionLog, runReadyBarrier, }); + // Argument serialization above can execute user code + // (getters, WORKFLOW_SERIALIZE hooks) in the live VM. + // Any randomness it drew would desync future + // correlation IDs from a fresh replay's, so demote + // the session to ordinary replay. + if ( + workflowExecution.type === 'retained' && + workflowExecution.session.randomDrawCount() !== + workflowExecution.drawCountAtSuspension + ) { + workflowExecution = { type: 'replay' }; + } } catch (suspensionError) { // A suspension create whose stale (412) rejection // survived the in-guard reload retries: schedule an diff --git a/packages/core/src/vm/index.ts b/packages/core/src/vm/index.ts index ba67168173..3b71fe03f5 100644 --- a/packages/core/src/vm/index.ts +++ b/packages/core/src/vm/index.ts @@ -47,7 +47,17 @@ function toDigestBytes(data: ArrayBuffer | ArrayBufferView): Uint8Array { export function createContext(options: CreateContextOptions) { let { fixedTimestamp } = options; const { seed } = options; - const rng = seedrandom(seed); + // Count every draw from the seeded stream (Math.random, getRandomValues, + // randomUUID, and the ULIDs/nanoids derived from them all flow through this + // one closure). The runtime compares the count across the suspension + // boundary to detect serialization side effects (see + // `canRetainWorkflowSession`). + const baseRng = seedrandom(seed); + let randomDrawCount = 0; + const rng = () => { + randomDrawCount++; + return baseRng(); + }; const context = vmCreateContext(); const g: typeof globalThis = runInContext('globalThis', context); @@ -208,5 +218,6 @@ export function createContext(options: CreateContextOptions) { fixedTimestamp = timestamp; }, usedHostAsync: () => usedHostAsync, + randomDrawCount: () => randomDrawCount, }; } diff --git a/packages/core/src/workflow.ts b/packages/core/src/workflow.ts index 5ff5991c41..c158f006d5 100644 --- a/packages/core/src/workflow.ts +++ b/packages/core/src/workflow.ts @@ -184,6 +184,14 @@ export interface WorkflowSession { * reconstructs — it must not be retained. */ usedHostAsync(): boolean; + /** + * Draws from the VM's seeded random stream so far. The runtime compares + * this across the suspension boundary: if post-suspension argument + * serialization (getters, `WORKFLOW_SERIALIZE` hooks) drew randomness, the + * retained VM's future correlation IDs would desync from what a fresh + * replay regenerates, so the session must not be retained. + */ + randomDrawCount(): number; resume(events: Event[]): WorkflowSessionResumeResult; } @@ -375,6 +383,7 @@ function createWorkflowSession({ globalThis: vmGlobalThis, updateTimestamp, usedHostAsync, + randomDrawCount, } = createContext({ seed: `${workflowRun.runId}:${workflowRun.workflowName}:${workflowRun.deploymentId}`, fixedTimestamp, @@ -1144,6 +1153,7 @@ function createWorkflowSession({ workflowRun, argumentCount: args.length, usedHostAsync, + randomDrawCount, resume(nextEvents) { switch (state.type) { case 'suspended': { From d83b651ad6dda2cb09c66f9afcccdfe56517f48b Mon Sep 17 00:00:00 2001 From: Nathan Colosimo <110621881+NathanColosimo@users.noreply.github.com> Date: Fri, 17 Jul 2026 13:23:40 -0700 Subject: [PATCH 09/43] refactor(core): make VM quiescence unconditional, cut tracking machinery MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Delete Atomics.waitAsync and the async WebAssembly entry points from the sandbox instead of tracking their use — with digest synchronous and GC intrinsics removed, no sandbox API settles a promise on host timing, so a suspended VM provably cannot advance. This deletes the trackHostAsync wrapper, the usedHostAsync bit and session method, the runtime gate clause, the session 'failed' state (unreachable), and the background-progress test scenarios (impossible by construction). --- .changeset/retain-workflow-vm.md | 2 +- .../v5/api-reference/workflow-globals.mdx | 2 + .../docs/v5/configuration/runtime-tuning.mdx | 2 +- packages/core/src/retained-vm-loop.test.ts | 23 --- packages/core/src/runtime.ts | 7 +- packages/core/src/vm/index.test.ts | 72 +++---- packages/core/src/vm/index.ts | 47 ++--- packages/core/src/workflow.test.ts | 176 +----------------- packages/core/src/workflow.ts | 22 +-- 9 files changed, 68 insertions(+), 285 deletions(-) diff --git a/.changeset/retain-workflow-vm.md b/.changeset/retain-workflow-vm.md index 61cb327ee9..b86e3d918c 100644 --- a/.changeset/retain-workflow-vm.md +++ b/.changeset/retain-workflow-vm.md @@ -3,4 +3,4 @@ 'workflow': patch --- -Retain workflow execution across inline steps within one invocation; `WORKFLOW_RETAINED_VM=0` disables retention. `crypto.subtle.digest` in workflow functions now computes synchronously via `node:crypto`, so digest timing is deterministic under replay. `WeakRef` and `FinalizationRegistry` are no longer exposed in workflow functions (GC observation cannot be replayed). +Retain workflow execution across inline steps within one invocation; `WORKFLOW_RETAINED_VM=0` disables retention. `crypto.subtle.digest` in workflow functions now computes synchronously via `node:crypto`, so digest timing is deterministic under replay. `WeakRef`, `FinalizationRegistry`, `Atomics.waitAsync`, and async `WebAssembly` compilation are no longer exposed in workflow functions (GC observation and wall-clock timing cannot be replayed). diff --git a/docs/content/docs/v5/api-reference/workflow-globals.mdx b/docs/content/docs/v5/api-reference/workflow-globals.mdx index 4f7eedf1d9..6faa7e3df6 100644 --- a/docs/content/docs/v5/api-reference/workflow-globals.mdx +++ b/docs/content/docs/v5/api-reference/workflow-globals.mdx @@ -101,3 +101,5 @@ The following are **not available** in workflow functions. Move this logic to [s - **Timers**: `setTimeout`, `setInterval`, `setImmediate`, and their `clear*` counterparts. Use [`sleep()`](/docs/api-reference/workflow/sleep) instead. See [timeout-in-workflow](/docs/errors/timeout-in-workflow). - **`Buffer`**: Node.js-specific API. Use `Uint8Array` with `toBase64()` / `fromBase64()` / `toHex()` / `fromHex()` for binary data encoding, or `atob()` / `btoa()` for string-based base64. - **`WeakRef` and `FinalizationRegistry`**: garbage-collection timing is not deterministic, so observing it would make workflow code impossible to replay. (`WeakMap` and `WeakSet` remain available — they do not expose GC state.) +- **`Atomics.waitAsync`**: a wall-clock timer, which cannot be replayed. Use [`sleep()`](/docs/api-reference/workflow/sleep) instead. +- **Async `WebAssembly` compilation** (`compile`, `instantiate`, `compileStreaming`, `instantiateStreaming`): resolves on compile-thread timing. The synchronous `new WebAssembly.Module()` and `new WebAssembly.Instance()` constructors remain available. diff --git a/docs/content/docs/v5/configuration/runtime-tuning.mdx b/docs/content/docs/v5/configuration/runtime-tuning.mdx index e5b65c5eaa..f2f1a98e53 100644 --- a/docs/content/docs/v5/configuration/runtime-tuning.mdx +++ b/docs/content/docs/v5/configuration/runtime-tuning.mdx @@ -79,7 +79,7 @@ For example, a workflow can run a 10-minute inline step even with `WORKFLOW_REPL - Default: enabled - Keeps the suspended workflow VM alive across inline steps within one invocation, so each iteration of the inline loop appends only the newly written events instead of replaying the whole event log in a fresh VM. -- Suspensions involving hooks, waits, or attributes, workflows that used host-timed async APIs (`Atomics.waitAsync`, async `WebAssembly` compilation), and any replay divergence always fall back to a full replay. +- Suspensions involving hooks, waits, or attributes — and any replay divergence — always fall back to a full replay. - Set `0` or `false` to replay from scratch in a fresh VM on every iteration. ### `WORKFLOW_INLINE_OWNERSHIP` diff --git a/packages/core/src/retained-vm-loop.test.ts b/packages/core/src/retained-vm-loop.test.ts index a76f58f8aa..84886896fb 100644 --- a/packages/core/src/retained-vm-loop.test.ts +++ b/packages/core/src/retained-vm-loop.test.ts @@ -40,18 +40,6 @@ const twoStepWorkflow = `const s1 = globalThis[Symbol.for("WORKFLOW_USE_STEP")]( } globalThis.__private_workflows = new Map([["workflow", workflow]]);`; -// `Atomics.waitAsync` resolves on host timing, not from the event log, so -// this VM can advance while suspended — the loop must decline retention. -const hostAsyncWorkflow = `const s1 = globalThis[Symbol.for("WORKFLOW_USE_STEP")]("r_s1"); - const s2 = globalThis[Symbol.for("WORKFLOW_USE_STEP")]("r_s2"); - async function workflow() { - await Atomics.waitAsync(new Int32Array(new SharedArrayBuffer(4)), 0, 1).value; - const a = await s1(); - const b = await s2(); - return a + b; - } - globalThis.__private_workflows = new Map([["workflow", workflow]]);`; - // Serializing this step's arguments executes the getter, which draws from // the VM's seeded random stream after the suspension — the loop must demote // the session so future correlation IDs stay replayable. @@ -210,17 +198,6 @@ describe('retained VM through the inline replay loop', () => { expect(on.output).toEqual(off.output); }); - it('declines retention for a VM that ran host-timed async work', async () => { - const { vmBuilds, output } = await drive( - 'wrun_retained_host_async', - hostAsyncWorkflow - ); - expect(output).toBeInstanceOf(Uint8Array); - // Atomics.waitAsync marks the VM as non-quiescent, so every iteration - // replays in a fresh VM even though retention is on by default. - expect(vmBuilds).toBeGreaterThan(1); - }); - it('demotes retention when argument serialization draws randomness', async () => { const { vmBuilds, output } = await drive( 'wrun_retained_impure_args', diff --git a/packages/core/src/runtime.ts b/packages/core/src/runtime.ts index bd29c8e93e..448cfd6fce 100644 --- a/packages/core/src/runtime.ts +++ b/packages/core/src/runtime.ts @@ -318,14 +318,11 @@ function openHookAndWaitState(events: Event[]): { /** * Retain only pure step boundaries with no out-of-band continuation source. - * Attributes require replay; hooks and waits can wake another invocation; a - * VM that ran host-timed async work can advance while suspended, so its live - * state may not match what a replay of the event log reconstructs. + * Attributes require replay; hooks and waits can wake another invocation. * `WORKFLOW_RETAINED_VM=0` disables retention entirely. */ function canRetainWorkflowSession( suspension: WorkflowSuspension, - session: WorkflowSession, events: Event[] ): boolean { return ( @@ -336,7 +333,6 @@ function canRetainWorkflowSession( suspension.attributeCount === 0 && suspension.hookDisposedCount === 0 && suspension.abortCount === 0 && - !session.usedHostAsync() && !hasOpenHookOrWait(events) ); } @@ -1503,7 +1499,6 @@ export function workflowEntrypoint( if (workflowResult.type === 'suspended') { workflowExecution = canRetainWorkflowSession( workflowResult.suspension, - workflowResult.session, events ) ? { diff --git a/packages/core/src/vm/index.test.ts b/packages/core/src/vm/index.test.ts index b2d694c5cd..ffd23eae66 100644 --- a/packages/core/src/vm/index.test.ts +++ b/packages/core/src/vm/index.test.ts @@ -258,56 +258,44 @@ describe('createContext', () => { }); }); -describe('usedHostAsync', () => { - it('starts false and ignores log-driven and deterministic APIs', () => { - const { context, usedHostAsync } = createContext({ seed, fixedTimestamp }); +describe('host-timed async', () => { + it('does not expose any API that settles on host timing', () => { + const { context } = createContext({ seed, fixedTimestamp }); - vm.runInContext( - 'Math.random(); Date.now(); crypto.randomUUID(); crypto.getRandomValues(new Uint8Array(4))', - context + expect(vm.runInContext('typeof Atomics.waitAsync', context)).toBe( + 'undefined' ); - expect(usedHostAsync()).toBe(false); - }); - - it('does not flip on the synchronous crypto.subtle.digest', async () => { - const { context, usedHostAsync } = createContext({ seed, fixedTimestamp }); - - await vm.runInContext( - 'crypto.subtle.digest("SHA-256", new Uint8Array(1))', - context + expect(vm.runInContext('typeof WebAssembly.compile', context)).toBe( + 'undefined' ); - expect(usedHostAsync()).toBe(false); - }); - - it('flips on Atomics.waitAsync', () => { - const { context, usedHostAsync } = createContext({ seed, fixedTimestamp }); - - vm.runInContext( - 'Atomics.waitAsync(new Int32Array(new SharedArrayBuffer(4)), 0, 1)', - context + expect(vm.runInContext('typeof WebAssembly.instantiate', context)).toBe( + 'undefined' ); - expect(usedHostAsync()).toBe(true); - }); - - it('flips on WebAssembly.compile', async () => { - const { context, usedHostAsync } = createContext({ seed, fixedTimestamp }); - - // Minimal valid empty module: magic + version. - await vm.runInContext( - 'WebAssembly.compile(new Uint8Array([0,97,115,109,1,0,0,0]))', - context + expect( + vm.runInContext('typeof WebAssembly.compileStreaming', context) + ).toBe('undefined'); + expect( + vm.runInContext('typeof WebAssembly.instantiateStreaming', context) + ).toBe('undefined'); + // Deterministic synchronous WebAssembly stays available. + expect(vm.runInContext('typeof WebAssembly.Module', context)).toBe( + 'function' ); - expect(usedHostAsync()).toBe(true); }); +}); - it('flips on WebAssembly.instantiate', async () => { - const { context, usedHostAsync } = createContext({ seed, fixedTimestamp }); +describe('randomDrawCount', () => { + it('counts every draw from the seeded stream', () => { + const { context, randomDrawCount } = createContext({ + seed, + fixedTimestamp, + }); - await vm.runInContext( - 'WebAssembly.instantiate(new Uint8Array([0,97,115,109,1,0,0,0]))', - context - ); - expect(usedHostAsync()).toBe(true); + expect(randomDrawCount()).toBe(0); + vm.runInContext('Math.random()', context); + expect(randomDrawCount()).toBe(1); + vm.runInContext('crypto.getRandomValues(new Uint8Array(4))', context); + expect(randomDrawCount()).toBe(5); }); }); diff --git a/packages/core/src/vm/index.ts b/packages/core/src/vm/index.ts index 3b71fe03f5..325862c5f8 100644 --- a/packages/core/src/vm/index.ts +++ b/packages/core/src/vm/index.ts @@ -95,36 +95,24 @@ export function createContext(options: CreateContextOptions) { const randomUUID = createRandomUUID(rng); - // Track every sandbox API whose promise resolves on host timing rather - // than from the event log: `Atomics.waitAsync` (a wall-clock timer via - // SharedArrayBuffer) and the async `WebAssembly` compilation entry points. - // These are the only ways a suspended workflow can make progress the event - // log cannot replay, so the runtime declines to retain a VM that used any - // of them (see `canRetainWorkflowSession`). Dynamic `import()` settles - // within a microtask (rejected: no `importModuleDynamically`), so it - // cannot advance a suspended VM. - let usedHostAsync = false; - const trackHostAsync = ( - target: Record, - method: string - ): void => { - const original = target[method]; - if (typeof original !== 'function') return; - target[method] = (...args: unknown[]) => { - usedHostAsync = true; - return Reflect.apply(original, target, args); - }; - }; + // The sandbox must not expose any way for workflow code to observe host + // timing or host state: after this block, every promise a workflow can + // create settles either from the event log or within its own microtask + // cascade, so a suspended VM is fully quiescent and can be retained across + // inline steps (see `canRetainWorkflowSession`). + // + // - `Atomics.waitAsync` is a wall-clock timer (via SharedArrayBuffer). + // - The async `WebAssembly` entry points resolve on compile-thread timing; + // the synchronous `new WebAssembly.Module()` / `Instance()` remain. + // - `WeakRef.deref()` and finalizer callbacks observe GC timing. + // - Dynamic `import()` settles within a microtask (rejected: no + // `importModuleDynamically`), so it needs no handling. const intrinsics = g as unknown as Record>; - trackHostAsync(intrinsics.Atomics, 'waitAsync'); - trackHostAsync(intrinsics.WebAssembly, 'compile'); - trackHostAsync(intrinsics.WebAssembly, 'instantiate'); - trackHostAsync(intrinsics.WebAssembly, 'compileStreaming'); - trackHostAsync(intrinsics.WebAssembly, 'instantiateStreaming'); - - // GC observation (`WeakRef.deref()`, finalizer callbacks) depends on host - // GC timing that neither replay nor a retained VM can reconstruct from the - // event log, so the sandbox does not expose it at all. + delete intrinsics.Atomics.waitAsync; + delete intrinsics.WebAssembly.compile; + delete intrinsics.WebAssembly.instantiate; + delete intrinsics.WebAssembly.compileStreaming; + delete intrinsics.WebAssembly.instantiateStreaming; delete intrinsics.WeakRef; delete intrinsics.FinalizationRegistry; @@ -217,7 +205,6 @@ export function createContext(options: CreateContextOptions) { updateTimestamp: (timestamp: number) => { fixedTimestamp = timestamp; }, - usedHostAsync: () => usedHostAsync, randomDrawCount: () => randomDrawCount, }; } diff --git a/packages/core/src/workflow.test.ts b/packages/core/src/workflow.test.ts index 2af9c6de2b..d7dfe39ab5 100644 --- a/packages/core/src/workflow.test.ts +++ b/packages/core/src/workflow.test.ts @@ -342,15 +342,15 @@ describe('runWorkflow', () => { log.mockRestore(); }); - it('returns a workflow that completes while its retained session is suspended', async () => { + it('falls back to replay permanently when the event prefix diverges', async () => { const ops: Promise[] = []; const workflowRun: WorkflowRun = { - runId: 'wrun_retained_async_completion', + runId: 'wrun_retained_divergence', workflowName: 'workflow', status: 'running', input: await dehydrateWorkflowArguments( [], - 'wrun_retained_async_completion', + 'wrun_retained_divergence', noEncryptionKey, ops ), @@ -361,17 +361,7 @@ describe('runWorkflow', () => { }; const workflowCode = ` const step = globalThis[Symbol.for("WORKFLOW_USE_STEP")]("step"); - async function waitRepeatedly() { - const shared = new Int32Array(new SharedArrayBuffer(4)); - for (let index = 0; index < 20; index++) { - await Atomics.waitAsync(shared, 0, 0, 25).value; - } - } - async function workflow() { - await Promise.race([step(), waitRepeatedly()]); - console.log("retained:background-completed"); - return "background completed"; - } + async function workflow() { await step(); } ${getWorkflowTransformCode('workflow')}`; const events: Event[] = [ { @@ -381,7 +371,6 @@ describe('runWorkflow', () => { createdAt: new Date('2024-01-01T00:00:00.000Z'), }, ]; - const log = vi.spyOn(console, 'log').mockImplementation(() => undefined); const suspended = await executeWorkflow({ type: 'replay', @@ -392,173 +381,28 @@ describe('runWorkflow', () => { replayPayloadCache: new ReplayPayloadCache(noEncryptionKey), }); assert(suspended.type === 'suspended'); - const pendingStep = suspended.suspension.steps[0]; - assert(pendingStep?.type === 'step'); - - const rewrittenSession = await executeWorkflow({ - type: 'replay', - workflowCode, - workflowRun, - events, - encryptionKey: noEncryptionKey, - replayPayloadCache: new ReplayPayloadCache(noEncryptionKey), - }); - assert(rewrittenSession.type === 'suspended'); - await vi.waitFor( - () => { - expect(log).toHaveBeenCalledTimes(2); - }, - { timeout: 10_000, interval: 10 } - ); - const rewrittenEvents = [ + // A log whose consumed prefix was rewritten is not a strict extension. + const rewritten = [ { ...events[0], eventId: 'rewritten-event' } as Event, { ...events[0], eventId: 'new-event' } as Event, ]; expect( await executeWorkflow({ type: 'resume', - session: rewrittenSession.session, - events: rewrittenEvents, - }) - ).toEqual({ type: 'replay' }); - - const createdAt = new Date('2024-01-01T00:00:01.000Z'); - events.push( - { - eventId: 'event-step-created', - runId: workflowRun.runId, - eventType: 'step_created', - correlationId: pendingStep.correlationId, - eventData: { stepName: pendingStep.stepName }, - createdAt, - }, - { - eventId: 'event-step-started', - runId: workflowRun.runId, - eventType: 'step_started', - correlationId: pendingStep.correlationId, - eventData: { stepName: pendingStep.stepName }, - createdAt, - }, - { - eventId: 'event-step-completed', - runId: workflowRun.runId, - eventType: 'step_completed', - correlationId: pendingStep.correlationId, - eventData: { - stepName: pendingStep.stepName, - result: await dehydrateStepReturnValue( - undefined, - workflowRun.runId, - noEncryptionKey, - ops - ), - }, - createdAt, - } - ); - - const completed = await executeWorkflow({ - type: 'resume', - session: suspended.session, - events, - }); - assert(completed.type === 'completed'); - expect( - await executeWorkflow({ - type: 'resume', - session: rewrittenSession.session, - events, + session: suspended.session, + events: rewritten, }) ).toEqual({ type: 'replay' }); - expect( - await hydrateWorkflowReturnValue( - completed.output as any, - workflowRun.runId, - noEncryptionKey, - ops - ) - ).toBe('background completed'); - log.mockRestore(); - }); - - it('does not drain operations from a discarded retained session', async () => { - const ops: Promise[] = []; - const workflowRun: WorkflowRun = { - runId: 'wrun_retained_discarded', - workflowName: 'workflow', - status: 'running', - input: await dehydrateWorkflowArguments( - [], - 'wrun_retained_discarded', - noEncryptionKey, - ops - ), - createdAt: new Date('2024-01-01T00:00:00.000Z'), - updatedAt: new Date('2024-01-01T00:00:00.000Z'), - startedAt: new Date('2024-01-01T00:00:00.000Z'), - deploymentId: 'test-deployment', - }; - const workflowCode = ` - const step = globalThis[Symbol.for("WORKFLOW_USE_STEP")]("step"); - const setAttributes = globalThis[Symbol.for("WORKFLOW_SET_ATTRIBUTES")]; - async function waitRepeatedly() { - const shared = new Int32Array(new SharedArrayBuffer(4)); - for (let index = 0; index < 20; index++) { - await Atomics.waitAsync(shared, 0, 0, 25).value; - } - } - async function workflow() { - await Promise.race([step(), waitRepeatedly()]); - void setAttributes([{ key: "stale", value: true }]); - console.log("retained:discarded-completed"); - return "discarded"; - } - ${getWorkflowTransformCode('workflow')}`; - const create = vi.fn(); - const log = vi.spyOn(console, 'log').mockImplementation(() => undefined); - setWorld({ - specVersion: SPEC_VERSION_CURRENT, - events: { create }, - streams: { write: vi.fn(), close: vi.fn() }, - } as any); - - const suspended = await executeWorkflow({ - type: 'replay', - workflowCode, - workflowRun, - events: [], - encryptionKey: noEncryptionKey, - replayPayloadCache: new ReplayPayloadCache(noEncryptionKey), - }); - assert(suspended.type === 'suspended'); - - await vi.waitFor( - () => { - expect(log).toHaveBeenCalledWith('retained:discarded-completed'); - }, - { timeout: 10_000, interval: 10 } - ); - await new Promise((resolve) => setTimeout(resolve, 250)); + // The fallback is permanent, even for a well-formed extension. expect( await executeWorkflow({ type: 'resume', session: suspended.session, - events: [ - { - eventId: 'event-run-created', - runId: workflowRun.runId, - eventType: 'run_created', - createdAt: new Date('2024-01-01T00:00:00.000Z'), - }, - ], + events, }) ).toEqual({ type: 'replay' }); - expect(create).not.toHaveBeenCalled(); - setWorld(undefined); - log.mockRestore(); }); it('regenerates step correlation IDs independent of startedAt (turbo replay-stability)', async () => { diff --git a/packages/core/src/workflow.ts b/packages/core/src/workflow.ts index c158f006d5..35ee9313a3 100644 --- a/packages/core/src/workflow.ts +++ b/packages/core/src/workflow.ts @@ -148,7 +148,6 @@ type WorkflowSessionState = readonly interruption: PromiseWithResolvers; } | { readonly type: 'suspended'; readonly suspension: WorkflowSuspension } - | { readonly type: 'failed'; readonly error: Error } | { readonly type: 'replay' } | { readonly type: 'completed' }; @@ -164,12 +163,15 @@ function isSameSuspensionBoundary( ); } +// A suspended VM is quiescent, but each parked step consumer signals its own +// (identical) suspension via scheduleWhenIdle — absorb those duplicates and +// treat anything else as unretainable. function updateSuspendedSession( suspension: WorkflowSuspension, error: Error ): WorkflowSessionState { - if (!WorkflowSuspension.is(error)) return { type: 'failed', error }; - return isSameSuspensionBoundary(suspension, error) + return WorkflowSuspension.is(error) && + isSameSuspensionBoundary(suspension, error) ? { type: 'suspended', suspension } : { type: 'replay' }; } @@ -177,13 +179,6 @@ function updateSuspendedSession( export interface WorkflowSession { readonly workflowRun: WorkflowRun; readonly argumentCount: number; - /** - * Whether the VM ran host-timed async work (`Atomics.waitAsync`, async - * `WebAssembly` compilation). Such a VM can advance while suspended, so its - * live state may diverge from what a replay of the durable event log - * reconstructs — it must not be retained. - */ - usedHostAsync(): boolean; /** * Draws from the VM's seeded random stream so far. The runtime compares * this across the suspension boundary: if post-suspension argument @@ -382,7 +377,6 @@ function createWorkflowSession({ context, globalThis: vmGlobalThis, updateTimestamp, - usedHostAsync, randomDrawCount, } = createContext({ seed: `${workflowRun.runId}:${workflowRun.workflowName}:${workflowRun.deploymentId}`, @@ -401,14 +395,13 @@ function createWorkflowSession({ const { interruption } = state; state = WorkflowSuspension.is(error) ? { type: 'suspended', suspension: error } - : { type: 'failed', error }; + : { type: 'replay' }; interruption.reject(error); return; } case 'suspended': state = updateSuspendedSession(state.suspension, error); return; - case 'failed': case 'replay': case 'completed': return; @@ -1152,7 +1145,6 @@ function createWorkflowSession({ const session: WorkflowSession = { workflowRun, argumentCount: args.length, - usedHostAsync, randomDrawCount, resume(nextEvents) { switch (state.type) { @@ -1175,8 +1167,6 @@ function createWorkflowSession({ execution: waitForExecution(interruption), }; } - case 'failed': - return { type: 'resumed', execution: failWorkflow(state.error) }; case 'replay': return { type: 'replay' }; case 'completed': From b2c33d2645aa75f2eddc59d30db5c363fcbd3ec8 Mon Sep 17 00:00:00 2001 From: Nathan Colosimo <110621881+NathanColosimo@users.noreply.github.com> Date: Fri, 17 Jul 2026 13:54:39 -0700 Subject: [PATCH 10/43] refactor(core): gate retention on passively cloneable step inputs Replace the RNG draw-counter demotion with prevention: when a session is a retention candidate, new step inputs take a passive descriptor walk (never invoking getters; proxies, accessors, functions, custom classes, and platform wrappers decline) and safe values are structuredClone'd into the host realm before dehydration, so serialization never executes workflow-owned code against a retained VM. Unsafe inputs serialize the old way and the session falls back to ordinary replay. --- .changeset/retain-workflow-vm.md | 2 +- .../docs/v5/configuration/runtime-tuning.mdx | 1 + packages/core/src/retained-vm-loop.test.ts | 116 +++++++++-- packages/core/src/runtime.ts | 13 +- .../src/runtime/retained-step-input.test.ts | 140 +++++++++++++ .../core/src/runtime/retained-step-input.ts | 193 ++++++++++++++++++ .../core/src/runtime/suspension-handler.ts | 53 ++++- packages/core/src/vm/index.test.ts | 15 -- packages/core/src/vm/index.ts | 13 +- packages/core/src/workflow.ts | 10 - 10 files changed, 489 insertions(+), 67 deletions(-) create mode 100644 packages/core/src/runtime/retained-step-input.test.ts create mode 100644 packages/core/src/runtime/retained-step-input.ts diff --git a/.changeset/retain-workflow-vm.md b/.changeset/retain-workflow-vm.md index b86e3d918c..9949f21f78 100644 --- a/.changeset/retain-workflow-vm.md +++ b/.changeset/retain-workflow-vm.md @@ -3,4 +3,4 @@ 'workflow': patch --- -Retain workflow execution across inline steps within one invocation; `WORKFLOW_RETAINED_VM=0` disables retention. `crypto.subtle.digest` in workflow functions now computes synchronously via `node:crypto`, so digest timing is deterministic under replay. `WeakRef`, `FinalizationRegistry`, `Atomics.waitAsync`, and async `WebAssembly` compilation are no longer exposed in workflow functions (GC observation and wall-clock timing cannot be replayed). +Retain workflow execution across passive-data inline steps within one invocation; boundaries whose argument serialization could execute workflow code fall back to replay, and `WORKFLOW_RETAINED_VM=0` disables retention. `crypto.subtle.digest` in workflow functions now computes synchronously via `node:crypto`, so digest timing is deterministic under replay. `WeakRef`, `FinalizationRegistry`, `Atomics.waitAsync`, and async `WebAssembly` compilation are no longer exposed in workflow functions (GC observation and wall-clock timing cannot be replayed). diff --git a/docs/content/docs/v5/configuration/runtime-tuning.mdx b/docs/content/docs/v5/configuration/runtime-tuning.mdx index f2f1a98e53..9c275ef8f0 100644 --- a/docs/content/docs/v5/configuration/runtime-tuning.mdx +++ b/docs/content/docs/v5/configuration/runtime-tuning.mdx @@ -80,6 +80,7 @@ For example, a workflow can run a 10-minute inline step even with `WORKFLOW_REPL - Default: enabled - Keeps the suspended workflow VM alive across inline steps within one invocation, so each iteration of the inline loop appends only the newly written events instead of replaying the whole event log in a fresh VM. - Suspensions involving hooks, waits, or attributes — and any replay divergence — always fall back to a full replay. +- Step inputs containing accessors, proxies, functions, custom class serializers, or other values whose serialization could execute workflow code also fall back to replay. Plain data and standard in-memory collections remain retainable. - Set `0` or `false` to replay from scratch in a fresh VM on every iteration. ### `WORKFLOW_INLINE_OWNERSHIP` diff --git a/packages/core/src/retained-vm-loop.test.ts b/packages/core/src/retained-vm-loop.test.ts index 84886896fb..532f981f79 100644 --- a/packages/core/src/retained-vm-loop.test.ts +++ b/packages/core/src/retained-vm-loop.test.ts @@ -15,10 +15,13 @@ vi.mock('./vm/index.js', async (importActual) => { }); const { createContext } = await import('./vm/index.js'); +const { registerSerializationClass } = await import('./class-serialization.js'); const { registerStepFunction } = await import('./private.js'); const { setWorld } = await import('./runtime/world.js'); const { workflowEntrypoint } = await import('./runtime.js'); -const { dehydrateWorkflowArguments } = await import('./serialization.js'); +const { dehydrateWorkflowArguments, hydrateWorkflowReturnValue } = await import( + './serialization.js' +); vi.mock('@vercel/functions', () => ({ waitUntil: vi.fn((p: Promise) => { @@ -40,18 +43,50 @@ const twoStepWorkflow = `const s1 = globalThis[Symbol.for("WORKFLOW_USE_STEP")]( } globalThis.__private_workflows = new Map([["workflow", workflow]]);`; -// Serializing this step's arguments executes the getter, which draws from -// the VM's seeded random stream after the suspension — the loop must demote -// the session so future correlation IDs stay replayable. +// Serializing this step's arguments executes the getter after suspension. Its +// state mutation is not reconstructed by a cold replay, so this boundary must +// demote even though it does not draw randomness. const impureArgsWorkflow = `const s1 = globalThis[Symbol.for("WORKFLOW_USE_STEP")]("r_s1"); - const s2 = globalThis[Symbol.for("WORKFLOW_USE_STEP")]("r_s2"); + const echo = globalThis[Symbol.for("WORKFLOW_USE_STEP")]("r_echo"); async function workflow() { - const a = await s1({ get x() { Math.random(); return 1; } }); - const b = await s2(); - return a + b; + let counter = 0; + await s1({ get x() { counter++; return 1; } }); + return await echo(counter); + } + globalThis.__private_workflows = new Map([["workflow", workflow]]);`; + +const impureSerializerWorkflow = `const s1 = globalThis[Symbol.for("WORKFLOW_USE_STEP")]("r_s1"); + const echo = globalThis[Symbol.for("WORKFLOW_USE_STEP")]("r_echo"); + class Value { + static classId = "test/RetainedSerializerValue"; + static [Symbol.for("workflow-serialize")](instance) { + instance.onSerialize(); + return { value: instance.value }; + } + constructor(value, onSerialize) { + this.value = value; + this.onSerialize = onSerialize; + } + } + async function workflow() { + let counter = 0; + await s1(new Value(1, () => counter++)); + return await echo(counter); } globalThis.__private_workflows = new Map([["workflow", workflow]]);`; +class RetainedSerializerValue { + constructor(readonly value: number) {} + + static [Symbol.for('workflow-deserialize')](data: { value: number }) { + return new RetainedSerializerValue(data.value); + } +} +registerSerializationClass( + 'test/RetainedSerializerValue', + RetainedSerializerValue +); + // `crypto.subtle.digest` computes synchronously via node:crypto, so a // digest-using VM stays quiescent at suspension and remains retainable. const digestWorkflow = `const s1 = globalThis[Symbol.for("WORKFLOW_USE_STEP")]("r_s1"); @@ -66,6 +101,7 @@ const digestWorkflow = `const s1 = globalThis[Symbol.for("WORKFLOW_USE_STEP")](" registerStepFunction('r_s1', async () => 10); registerStepFunction('r_s2', async () => 20); +registerStepFunction('r_echo', async (value) => value); // Drive the full workflow handler over a stateful (dynamic) event log so the // inline loop makes real progress across its own writes, exactly like a World. @@ -198,13 +234,67 @@ describe('retained VM through the inline replay loop', () => { expect(on.output).toEqual(off.output); }); - it('demotes retention when argument serialization draws randomness', async () => { - const { vmBuilds, output } = await drive( - 'wrun_retained_impure_args', + it('matches cold replay when argument serialization mutates workflow state', async () => { + process.env.WORKFLOW_RETAINED_VM = '0'; + const off = await drive( + 'wrun_retained_impure_args_off', impureArgsWorkflow ); - expect(output).toBeInstanceOf(Uint8Array); - expect(vmBuilds).toBeGreaterThan(1); + createContextSpy.mockClear(); + + delete process.env.WORKFLOW_RETAINED_VM; + const on = await drive('wrun_retained_impure_args_on', impureArgsWorkflow); + + expect(on.vmBuilds).toBeGreaterThan(1); + expect( + await hydrateWorkflowReturnValue( + off.output, + 'wrun_retained_impure_args_off', + undefined, + [] + ) + ).toBe(0); + expect( + await hydrateWorkflowReturnValue( + on.output, + 'wrun_retained_impure_args_on', + undefined, + [] + ) + ).toBe(0); + }); + + it('matches cold replay when a custom serializer mutates workflow state', async () => { + process.env.WORKFLOW_RETAINED_VM = '0'; + const off = await drive( + 'wrun_retained_impure_serializer_off', + impureSerializerWorkflow + ); + createContextSpy.mockClear(); + + delete process.env.WORKFLOW_RETAINED_VM; + const on = await drive( + 'wrun_retained_impure_serializer_on', + impureSerializerWorkflow + ); + + expect(on.vmBuilds).toBeGreaterThan(1); + expect( + await hydrateWorkflowReturnValue( + off.output, + 'wrun_retained_impure_serializer_off', + undefined, + [] + ) + ).toBe(0); + expect( + await hydrateWorkflowReturnValue( + on.output, + 'wrun_retained_impure_serializer_on', + undefined, + [] + ) + ).toBe(0); }); it('retains a VM that used the synchronous crypto.subtle.digest', async () => { diff --git a/packages/core/src/runtime.ts b/packages/core/src/runtime.ts index 448cfd6fce..05f8f09795 100644 --- a/packages/core/src/runtime.ts +++ b/packages/core/src/runtime.ts @@ -1136,7 +1136,6 @@ export function workflowEntrypoint( | { readonly type: 'retained'; readonly session: WorkflowSession; - readonly drawCountAtSuspension: number; } = { type: 'replay' }; // Main replay loop @@ -1504,8 +1503,6 @@ export function workflowEntrypoint( ? { type: 'retained', session: workflowResult.session, - drawCountAtSuspension: - workflowResult.session.randomDrawCount(), } : { type: 'replay' }; throw workflowResult.suspension; @@ -1641,16 +1638,12 @@ export function workflowEntrypoint( requestId, eventLog: suspensionLog, runReadyBarrier, + prepareForRetention: + workflowExecution.type === 'retained', }); - // Argument serialization above can execute user code - // (getters, WORKFLOW_SERIALIZE hooks) in the live VM. - // Any randomness it drew would desync future - // correlation IDs from a fresh replay's, so demote - // the session to ordinary replay. if ( workflowExecution.type === 'retained' && - workflowExecution.session.randomDrawCount() !== - workflowExecution.drawCountAtSuspension + !suspensionResult.retainedStepInputsSafe ) { workflowExecution = { type: 'replay' }; } diff --git a/packages/core/src/runtime/retained-step-input.test.ts b/packages/core/src/runtime/retained-step-input.test.ts new file mode 100644 index 0000000000..0b8e892cd1 --- /dev/null +++ b/packages/core/src/runtime/retained-step-input.test.ts @@ -0,0 +1,140 @@ +import vm from 'node:vm'; +import { describe, expect, it } from 'vitest'; +import * as stepSerialization from '../serialization/step.js'; +import { createContext } from '../vm/index.js'; +import { prepareRetainedStepInput } from './retained-step-input.js'; + +const seed = 'retained-step-input'; +const fixedTimestamp = 1_700_000_000_000; + +describe('prepareRetainedStepInput', () => { + it('clones passive cross-realm data into the host realm', () => { + const { context, globalThis: workflowGlobal } = createContext({ + seed, + fixedTimestamp, + }); + const value = vm.runInContext( + `(() => { + const buffer = new ArrayBuffer(8); + return { + nested: [{ ok: true }], + map: new Map([["key", new Set([1, 2])]]), + bytes: new Uint8Array(buffer, 2, 4), + }; + })()`, + context + ); + + const prepared = prepareRetainedStepInput(value, workflowGlobal); + + expect(prepared.retainable).toBe(true); + if (!prepared.retainable) return; + expect(prepared.value).toEqual({ + nested: [{ ok: true }], + map: new Map([['key', new Set([1, 2])]]), + bytes: new Uint8Array(4), + }); + expect(Object.getPrototypeOf(prepared.value)).toBe(Object.prototype); + }); + + it('produces the same serialized bytes as ordinary VM traversal', async () => { + const { context, globalThis: workflowGlobal } = createContext({ + seed, + fixedTimestamp, + }); + const value = vm.runInContext( + `({ + nested: [{ ok: true, missing: undefined }], + map: new Map([["key", new Set([1, 2])]]), + bytes: new Uint8Array([1, 2, 3, 4]), + date: new Date(1234), + regexp: /workflow/gi, + })`, + context + ); + const prepared = prepareRetainedStepInput(value, workflowGlobal); + expect(prepared.retainable).toBe(true); + if (!prepared.retainable) return; + + const original = await stepSerialization.serialize(value, undefined, { + global: workflowGlobal, + }); + const cloned = await stepSerialization.serialize( + prepared.value, + undefined, + { global: globalThis } + ); + + expect(cloned).toEqual(original); + }); + + it('declines accessors without invoking them', () => { + const { context, globalThis: workflowGlobal } = createContext({ + seed, + fixedTimestamp, + }); + const value = vm.runInContext( + `(() => { + globalThis.__retainedTestCalls = 0; + return { get value() { globalThis.__retainedTestCalls++; return 1; } }; + })()`, + context + ); + + expect(prepareRetainedStepInput(value, workflowGlobal)).toEqual({ + retainable: false, + }); + expect(vm.runInContext('globalThis.__retainedTestCalls', context)).toBe(0); + }); + + it('declines proxies without invoking their traps', () => { + const { context, globalThis: workflowGlobal } = createContext({ + seed, + fixedTimestamp, + }); + const value = vm.runInContext( + `(() => { + globalThis.__retainedTestCalls = 0; + return new Proxy({ value: 1 }, { + ownKeys(target) { + globalThis.__retainedTestCalls++; + return Reflect.ownKeys(target); + } + }); + })()`, + context + ); + + expect(prepareRetainedStepInput(value, workflowGlobal)).toEqual({ + retainable: false, + }); + expect(vm.runInContext('globalThis.__retainedTestCalls', context)).toBe(0); + }); + + it('declines custom class serializers without invoking them', () => { + const { context, globalThis: workflowGlobal } = createContext({ + seed, + fixedTimestamp, + }); + const value = vm.runInContext( + `(() => { + globalThis.__retainedTestCalls = 0; + class Value { + static classId = "test/Value"; + static [Symbol.for("workflow-serialize")](instance) { + globalThis.__retainedTestCalls++; + return { value: instance.value }; + } + constructor(value) { this.value = value; } + } + return new Value(1); + })()`, + context + ); + + expect(prepareRetainedStepInput(value, workflowGlobal)).toEqual({ + retainable: false, + }); + expect(vm.runInContext('globalThis.__retainedTestCalls', context)).toBe(0); + }); +}); diff --git a/packages/core/src/runtime/retained-step-input.ts b/packages/core/src/runtime/retained-step-input.ts new file mode 100644 index 0000000000..1440dd4728 --- /dev/null +++ b/packages/core/src/runtime/retained-step-input.ts @@ -0,0 +1,193 @@ +import { types } from 'node:util'; + +export type RetainedStepInputPreparation = + | { readonly retainable: true; readonly value: unknown } + | { readonly retainable: false }; + +const typedArrayNames = [ + 'BigInt64Array', + 'BigUint64Array', + 'Float16Array', + 'Float32Array', + 'Float64Array', + 'Int8Array', + 'Int16Array', + 'Int32Array', + 'Uint8Array', + 'Uint8ClampedArray', + 'Uint16Array', + 'Uint32Array', +] as const; + +function hasAllowedPrototype( + value: object, + workflowGlobal: Record, + constructorName: string +): boolean { + const prototype = Object.getPrototypeOf(value); + return ( + prototype === + globalThis[constructorName as keyof typeof globalThis]?.prototype || + prototype === workflowGlobal[constructorName]?.prototype + ); +} + +function isArrayIndex(key: string): boolean { + const index = Number(key); + return ( + Number.isInteger(index) && + index >= 0 && + index < 2 ** 32 - 1 && + String(index) === key + ); +} + +/** + * Whether structured cloning `value` can avoid executing workflow-owned code. + * + * The retained VM must not observe serialization side effects that a later + * cold replay cannot reconstruct. Proxy traps, accessors, functions, custom + * classes, and platform wrappers can all execute workflow code while devalue + * traverses them, so they deliberately decline the fast path. Plain data and + * standard in-memory collections are cloned into the host realm before + * serialization, keeping even mutable VM prototypes out of the write path. + */ +function isPassivelyCloneable( + value: unknown, + workflowGlobal: Record, + seen: WeakSet +): boolean { + if ( + value === null || + value === undefined || + typeof value === 'boolean' || + typeof value === 'bigint' || + typeof value === 'number' || + typeof value === 'string' + ) { + return true; + } + if (typeof value !== 'object' || types.isProxy(value)) return false; + if (seen.has(value)) return true; + seen.add(value); + + if (Array.isArray(value)) { + if (!hasAllowedPrototype(value, workflowGlobal, 'Array')) return false; + for (const key of Reflect.ownKeys(value)) { + if (key === 'length') continue; + const descriptor = Object.getOwnPropertyDescriptor(value, key); + if (!descriptor) return false; + if (key === 'then' || key === Symbol.toStringTag) return false; + if (typeof key !== 'string' || !isArrayIndex(key)) { + if (descriptor.enumerable) return false; + continue; + } + if (!('value' in descriptor)) return false; + if (!isPassivelyCloneable(descriptor.value, workflowGlobal, seen)) { + return false; + } + } + return true; + } + + if (types.isMap(value)) { + if (!hasAllowedPrototype(value, workflowGlobal, 'Map')) return false; + if (Reflect.ownKeys(value).length > 0) return false; + let retainable = true; + Map.prototype.forEach.call(value, (entryValue: unknown, key: unknown) => { + retainable &&= + isPassivelyCloneable(key, workflowGlobal, seen) && + isPassivelyCloneable(entryValue, workflowGlobal, seen); + }); + return retainable; + } + + if (types.isSet(value)) { + if (!hasAllowedPrototype(value, workflowGlobal, 'Set')) return false; + if (Reflect.ownKeys(value).length > 0) return false; + let retainable = true; + Set.prototype.forEach.call(value, (entryValue: unknown) => { + retainable &&= isPassivelyCloneable(entryValue, workflowGlobal, seen); + }); + return retainable; + } + + if (types.isDate(value)) { + return ( + hasAllowedPrototype(value, workflowGlobal, 'Date') && + Reflect.ownKeys(value).length === 0 + ); + } + if (types.isRegExp(value)) { + return ( + hasAllowedPrototype(value, workflowGlobal, 'RegExp') && + Reflect.ownKeys(value).every((key) => key === 'lastIndex') + ); + } + if (types.isArrayBuffer(value)) { + return ( + hasAllowedPrototype(value, workflowGlobal, 'ArrayBuffer') && + Reflect.ownKeys(value).length === 0 + ); + } + if (types.isSharedArrayBuffer(value)) return false; + if (types.isDataView(value)) { + return ( + hasAllowedPrototype(value, workflowGlobal, 'DataView') && + Reflect.ownKeys(value).length === 0 + ); + } + if (types.isTypedArray(value)) { + return ( + Reflect.ownKeys(value).every( + (key) => typeof key === 'string' && isArrayIndex(key) + ) && + typedArrayNames.some( + (name) => + workflowGlobal[name] !== undefined && + hasAllowedPrototype(value, workflowGlobal, name) + ) + ); + } + + const prototype = Object.getPrototypeOf(value); + if ( + prototype !== Object.prototype && + prototype !== workflowGlobal.Object?.prototype + ) { + return false; + } + + for (const key of Reflect.ownKeys(value)) { + const descriptor = Object.getOwnPropertyDescriptor(value, key); + if (!descriptor) return false; + if (typeof key === 'symbol') { + if (key === Symbol.toStringTag) return false; + if (descriptor.enumerable) return false; + continue; + } + if (!descriptor.enumerable) { + if (key === 'constructor' || key === 'then') return false; + continue; + } + if (key === '__proto__' || !('value' in descriptor)) return false; + if (!isPassivelyCloneable(descriptor.value, workflowGlobal, seen)) { + return false; + } + } + return true; +} + +export function prepareRetainedStepInput( + value: unknown, + workflowGlobal: Record +): RetainedStepInputPreparation { + if (!isPassivelyCloneable(value, workflowGlobal, new WeakSet())) { + return { retainable: false }; + } + try { + return { retainable: true, value: structuredClone(value) }; + } catch { + return { retainable: false }; + } +} diff --git a/packages/core/src/runtime/suspension-handler.ts b/packages/core/src/runtime/suspension-handler.ts index 291a45f57d..e0c35f63ef 100644 --- a/packages/core/src/runtime/suspension-handler.ts +++ b/packages/core/src/runtime/suspension-handler.ts @@ -31,6 +31,7 @@ import * as Attribute from '../telemetry/semantic-conventions.js'; import { getAbortStreamIdFromToken } from '../util.js'; import { getMaxInlineSteps } from './constants.js'; import { type MutableEventLog, withPreconditionRetry } from './helpers.js'; +import { prepareRetainedStepInput } from './retained-step-input.js'; export interface SuspensionHandlerParams { suspension: WorkflowSuspension; @@ -56,6 +57,11 @@ export interface SuspensionHandlerParams { * where `run_started` was already awaited up front. */ runReadyBarrier?: Promise; + /** + * Prepare new step inputs without traversing workflow-owned objects during + * serialization. Enabled only while the caller is holding a retained VM. + */ + prepareForRetention?: boolean; } /** @@ -121,6 +127,8 @@ export interface SuspensionHandlerResult { * durably creating the user's hooks doesn't count as runtime overhead. */ hookCreationMs: number; + /** Whether every newly serialized step input was passive retained-VM data. */ + retainedStepInputsSafe: boolean; } async function createHookEvent({ @@ -205,6 +213,7 @@ export async function handleSuspension({ requestId, eventLog, runReadyBarrier, + prepareForRetention = false, }: SuspensionHandlerParams): Promise { const runId = run.runId; @@ -488,6 +497,35 @@ export async function handleSuspension({ // racing with concurrent handlers on step execution. const createdStepCorrelationIds = new Set(); + let retainedStepInputsSafe = true; + const inputsByCorrelationId = new Map< + string, + { value: unknown; global: Record } + >(); + for (const queueItem of stepItems) { + if (!stepsNeedingCreation.has(queueItem.correlationId)) continue; + const value = { + args: queueItem.args, + closureVars: queueItem.closureVars, + thisVal: queueItem.thisVal, + }; + if (prepareForRetention) { + const prepared = prepareRetainedStepInput(value, suspension.globalThis); + if (prepared.retainable) { + inputsByCorrelationId.set(queueItem.correlationId, { + value: prepared.value, + global: globalThis, + }); + continue; + } + retainedStepInputsSafe = false; + } + inputsByCorrelationId.set(queueItem.correlationId, { + value, + global: suspension.globalThis, + }); + } + // Lazy inline start: defer the step_created write for up to // `getMaxInlineSteps()` steps the caller will run inline (in parallel). Each // step is created on the fly by the lazy `step_started` executeStep sends @@ -521,15 +559,17 @@ export async function handleSuspension({ if (stepsNeedingCreation.has(queueItem.correlationId)) { ops.push( (async () => { + const input = inputsByCorrelationId.get(queueItem.correlationId); + if (!input) { + throw new Error( + `Missing prepared input for step ${queueItem.correlationId}` + ); + } const dehydratedInput = await dehydrateStepArguments( - { - args: queueItem.args, - closureVars: queueItem.closureVars, - thisVal: queueItem.thisVal, - }, + input.value, runId, encryptionKey, - suspension.globalThis, + input.global, false, compression ); @@ -716,6 +756,7 @@ export async function handleSuspension({ hasAttributeEvents: attributeItems.length > 0, hasHookEvents: hooksNeedingCreation.length > 0, hookCreationMs, + retainedStepInputsSafe, }; } diff --git a/packages/core/src/vm/index.test.ts b/packages/core/src/vm/index.test.ts index ffd23eae66..677a5ba1f1 100644 --- a/packages/core/src/vm/index.test.ts +++ b/packages/core/src/vm/index.test.ts @@ -284,21 +284,6 @@ describe('host-timed async', () => { }); }); -describe('randomDrawCount', () => { - it('counts every draw from the seeded stream', () => { - const { context, randomDrawCount } = createContext({ - seed, - fixedTimestamp, - }); - - expect(randomDrawCount()).toBe(0); - vm.runInContext('Math.random()', context); - expect(randomDrawCount()).toBe(1); - vm.runInContext('crypto.getRandomValues(new Uint8Array(4))', context); - expect(randomDrawCount()).toBe(5); - }); -}); - describe('crypto.subtle.digest', () => { it.each([ 'SHA-1', diff --git a/packages/core/src/vm/index.ts b/packages/core/src/vm/index.ts index 325862c5f8..03bc34d2d7 100644 --- a/packages/core/src/vm/index.ts +++ b/packages/core/src/vm/index.ts @@ -47,17 +47,7 @@ function toDigestBytes(data: ArrayBuffer | ArrayBufferView): Uint8Array { export function createContext(options: CreateContextOptions) { let { fixedTimestamp } = options; const { seed } = options; - // Count every draw from the seeded stream (Math.random, getRandomValues, - // randomUUID, and the ULIDs/nanoids derived from them all flow through this - // one closure). The runtime compares the count across the suspension - // boundary to detect serialization side effects (see - // `canRetainWorkflowSession`). - const baseRng = seedrandom(seed); - let randomDrawCount = 0; - const rng = () => { - randomDrawCount++; - return baseRng(); - }; + const rng = seedrandom(seed); const context = vmCreateContext(); const g: typeof globalThis = runInContext('globalThis', context); @@ -205,6 +195,5 @@ export function createContext(options: CreateContextOptions) { updateTimestamp: (timestamp: number) => { fixedTimestamp = timestamp; }, - randomDrawCount: () => randomDrawCount, }; } diff --git a/packages/core/src/workflow.ts b/packages/core/src/workflow.ts index 35ee9313a3..83f25823b7 100644 --- a/packages/core/src/workflow.ts +++ b/packages/core/src/workflow.ts @@ -179,14 +179,6 @@ function updateSuspendedSession( export interface WorkflowSession { readonly workflowRun: WorkflowRun; readonly argumentCount: number; - /** - * Draws from the VM's seeded random stream so far. The runtime compares - * this across the suspension boundary: if post-suspension argument - * serialization (getters, `WORKFLOW_SERIALIZE` hooks) drew randomness, the - * retained VM's future correlation IDs would desync from what a fresh - * replay regenerates, so the session must not be retained. - */ - randomDrawCount(): number; resume(events: Event[]): WorkflowSessionResumeResult; } @@ -377,7 +369,6 @@ function createWorkflowSession({ context, globalThis: vmGlobalThis, updateTimestamp, - randomDrawCount, } = createContext({ seed: `${workflowRun.runId}:${workflowRun.workflowName}:${workflowRun.deploymentId}`, fixedTimestamp, @@ -1145,7 +1136,6 @@ function createWorkflowSession({ const session: WorkflowSession = { workflowRun, argumentCount: args.length, - randomDrawCount, resume(nextEvents) { switch (state.type) { case 'suspended': { From 744f0b7c1078648fa2e56f6ecbc775b88d1460b0 Mon Sep 17 00:00:00 2001 From: Nathan Colosimo <110621881+NathanColosimo@users.noreply.github.com> Date: Fri, 17 Jul 2026 14:08:42 -0700 Subject: [PATCH 11/43] fix(core): harden the passive step-input walker - require enumerable on array index descriptors: structuredClone drops non-enumerable indices that devalue persists - read workflow globals and constructor prototypes via own-property descriptors only, so validation can never execute workflow-owned accessors on redefined globals --- .../src/runtime/retained-step-input.test.ts | 48 ++++ .../core/src/runtime/retained-step-input.ts | 237 +++++++++++------- .../core/src/runtime/suspension-handler.ts | 21 +- 3 files changed, 208 insertions(+), 98 deletions(-) diff --git a/packages/core/src/runtime/retained-step-input.test.ts b/packages/core/src/runtime/retained-step-input.test.ts index 0b8e892cd1..c316a42992 100644 --- a/packages/core/src/runtime/retained-step-input.test.ts +++ b/packages/core/src/runtime/retained-step-input.test.ts @@ -138,3 +138,51 @@ describe('prepareRetainedStepInput', () => { expect(vm.runInContext('globalThis.__retainedTestCalls', context)).toBe(0); }); }); + +describe('prepareRetainedStepInput review-hardening', () => { + it('declines non-enumerable array indices (structuredClone would drop them)', () => { + const { context, globalThis: workflowGlobal } = createContext({ + seed, + fixedTimestamp, + }); + const value = vm.runInContext( + `(() => { + const arr = [1, 2]; + Object.defineProperty(arr, "0", { value: 7, enumerable: false }); + return arr; + })()`, + context + ); + + expect(prepareRetainedStepInput(value, workflowGlobal)).toEqual({ + retainable: false, + }); + }); + + it('never performs a property Get on redefined workflow globals', () => { + const { context, globalThis: workflowGlobal } = createContext({ + seed, + fixedTimestamp, + }); + const value = vm.runInContext( + `(() => { + globalThis.__retainedTestCalls = 0; + const arr = [1]; + for (const name of ["Array", "Object"]) { + Object.defineProperty(globalThis, name, { + get() { globalThis.__retainedTestCalls++; throw new Error("boom"); }, + }); + } + return { arr }; + })()`, + context + ); + + // The vandalized globals make the VM-realm prototypes unverifiable, so + // validation declines — without throwing or invoking the getters. + expect(prepareRetainedStepInput(value, workflowGlobal)).toEqual({ + retainable: false, + }); + expect(vm.runInContext('globalThis.__retainedTestCalls', context)).toBe(0); + }); +}); diff --git a/packages/core/src/runtime/retained-step-input.ts b/packages/core/src/runtime/retained-step-input.ts index 1440dd4728..129b1922be 100644 --- a/packages/core/src/runtime/retained-step-input.ts +++ b/packages/core/src/runtime/retained-step-input.ts @@ -19,17 +19,28 @@ const typedArrayNames = [ 'Uint32Array', ] as const; +// Own data-property read that never performs a property Get — workflow code +// can redefine its globals (or their `prototype` slots) with accessors, and +// validation must not execute workflow-owned code. +function ownDataProperty(target: object, key: string): unknown { + const descriptor = Object.getOwnPropertyDescriptor(target, key); + return descriptor && 'value' in descriptor ? descriptor.value : undefined; +} + function hasAllowedPrototype( value: object, workflowGlobal: Record, constructorName: string ): boolean { const prototype = Object.getPrototypeOf(value); - return ( - prototype === - globalThis[constructorName as keyof typeof globalThis]?.prototype || - prototype === workflowGlobal[constructorName]?.prototype - ); + const hostConstructor = globalThis[ + constructorName as keyof typeof globalThis + ] as { prototype?: object } | undefined; + if (hostConstructor !== undefined && prototype === hostConstructor.prototype) + return true; + const vmConstructor = ownDataProperty(workflowGlobal, constructorName); + if (typeof vmConstructor !== 'function') return false; + return prototype === ownDataProperty(vmConstructor, 'prototype'); } function isArrayIndex(key: string): boolean { @@ -42,76 +53,75 @@ function isArrayIndex(key: string): boolean { ); } -/** - * Whether structured cloning `value` can avoid executing workflow-owned code. - * - * The retained VM must not observe serialization side effects that a later - * cold replay cannot reconstruct. Proxy traps, accessors, functions, custom - * classes, and platform wrappers can all execute workflow code while devalue - * traverses them, so they deliberately decline the fast path. Plain data and - * standard in-memory collections are cloned into the host realm before - * serialization, keeping even mutable VM prototypes out of the write path. - */ -function isPassivelyCloneable( - value: unknown, +function isPassiveArrayProperty( + array: unknown[], + key: string | symbol, workflowGlobal: Record, seen: WeakSet ): boolean { - if ( - value === null || - value === undefined || - typeof value === 'boolean' || - typeof value === 'bigint' || - typeof value === 'number' || - typeof value === 'string' - ) { - return true; + if (key === 'length') return true; + const descriptor = Object.getOwnPropertyDescriptor(array, key); + if (!descriptor) return false; + if (key === 'then' || key === Symbol.toStringTag) return false; + if (typeof key !== 'string' || !isArrayIndex(key)) { + return !descriptor.enumerable; } - if (typeof value !== 'object' || types.isProxy(value)) return false; - if (seen.has(value)) return true; - seen.add(value); + // Non-enumerable indices are dropped by structuredClone but persisted by + // devalue, so they must decline the fast path. + return ( + descriptor.enumerable === true && + 'value' in descriptor && + isPassivelyCloneable(descriptor.value, workflowGlobal, seen) + ); +} - if (Array.isArray(value)) { - if (!hasAllowedPrototype(value, workflowGlobal, 'Array')) return false; - for (const key of Reflect.ownKeys(value)) { - if (key === 'length') continue; - const descriptor = Object.getOwnPropertyDescriptor(value, key); - if (!descriptor) return false; - if (key === 'then' || key === Symbol.toStringTag) return false; - if (typeof key !== 'string' || !isArrayIndex(key)) { - if (descriptor.enumerable) return false; - continue; - } - if (!('value' in descriptor)) return false; - if (!isPassivelyCloneable(descriptor.value, workflowGlobal, seen)) { - return false; - } - } - return true; - } +function isPassiveArray( + value: unknown[], + workflowGlobal: Record, + seen: WeakSet +): boolean { + return ( + hasAllowedPrototype(value, workflowGlobal, 'Array') && + Reflect.ownKeys(value).every((key) => + isPassiveArrayProperty(value, key, workflowGlobal, seen) + ) + ); +} - if (types.isMap(value)) { - if (!hasAllowedPrototype(value, workflowGlobal, 'Map')) return false; - if (Reflect.ownKeys(value).length > 0) return false; - let retainable = true; - Map.prototype.forEach.call(value, (entryValue: unknown, key: unknown) => { - retainable &&= - isPassivelyCloneable(key, workflowGlobal, seen) && - isPassivelyCloneable(entryValue, workflowGlobal, seen); - }); - return retainable; - } +function isPassiveMap( + value: Map, + workflowGlobal: Record, + seen: WeakSet +): boolean { + if (!hasAllowedPrototype(value, workflowGlobal, 'Map')) return false; + if (Reflect.ownKeys(value).length > 0) return false; + let retainable = true; + Map.prototype.forEach.call(value, (entryValue: unknown, key: unknown) => { + retainable &&= + isPassivelyCloneable(key, workflowGlobal, seen) && + isPassivelyCloneable(entryValue, workflowGlobal, seen); + }); + return retainable; +} - if (types.isSet(value)) { - if (!hasAllowedPrototype(value, workflowGlobal, 'Set')) return false; - if (Reflect.ownKeys(value).length > 0) return false; - let retainable = true; - Set.prototype.forEach.call(value, (entryValue: unknown) => { - retainable &&= isPassivelyCloneable(entryValue, workflowGlobal, seen); - }); - return retainable; - } +function isPassiveSet( + value: Set, + workflowGlobal: Record, + seen: WeakSet +): boolean { + if (!hasAllowedPrototype(value, workflowGlobal, 'Set')) return false; + if (Reflect.ownKeys(value).length > 0) return false; + let retainable = true; + Set.prototype.forEach.call(value, (entryValue: unknown) => { + retainable &&= isPassivelyCloneable(entryValue, workflowGlobal, seen); + }); + return retainable; +} +function isPassiveBuiltIn( + value: object, + workflowGlobal: Record +): boolean | undefined { if (types.isDate(value)) { return ( hasAllowedPrototype(value, workflowGlobal, 'Date') && @@ -142,40 +152,83 @@ function isPassivelyCloneable( Reflect.ownKeys(value).every( (key) => typeof key === 'string' && isArrayIndex(key) ) && - typedArrayNames.some( - (name) => - workflowGlobal[name] !== undefined && - hasAllowedPrototype(value, workflowGlobal, name) + typedArrayNames.some((name) => + hasAllowedPrototype(value, workflowGlobal, name) ) ); } + return undefined; +} - const prototype = Object.getPrototypeOf(value); +function isPassiveObjectProperty( + object: object, + key: string | symbol, + workflowGlobal: Record, + seen: WeakSet +): boolean { + const descriptor = Object.getOwnPropertyDescriptor(object, key); + if (!descriptor) return false; + if (typeof key === 'symbol') { + return key !== Symbol.toStringTag && !descriptor.enumerable; + } + if (!descriptor.enumerable) { + return key !== 'constructor' && key !== 'then'; + } + return ( + key !== '__proto__' && + 'value' in descriptor && + isPassivelyCloneable(descriptor.value, workflowGlobal, seen) + ); +} + +function isPassivePlainObject( + value: object, + workflowGlobal: Record, + seen: WeakSet +): boolean { + if (!hasAllowedPrototype(value, workflowGlobal, 'Object')) return false; + return Reflect.ownKeys(value).every((key) => + isPassiveObjectProperty(value, key, workflowGlobal, seen) + ); +} + +/** + * Whether structured cloning `value` can avoid executing workflow-owned code. + * + * The retained VM must not observe serialization side effects that a later + * cold replay cannot reconstruct. Proxy traps, accessors, functions, custom + * classes, and platform wrappers can all execute workflow code while devalue + * traverses them, so they deliberately decline the fast path. Plain data and + * standard in-memory collections are cloned into the host realm before + * serialization, keeping even mutable VM prototypes out of the write path. + */ +function isPassivelyCloneable( + value: unknown, + workflowGlobal: Record, + seen: WeakSet +): boolean { if ( - prototype !== Object.prototype && - prototype !== workflowGlobal.Object?.prototype + value === null || + value === undefined || + typeof value === 'boolean' || + typeof value === 'bigint' || + typeof value === 'number' || + typeof value === 'string' ) { - return false; + return true; } + if (typeof value !== 'object' || types.isProxy(value)) return false; + if (seen.has(value)) return true; + seen.add(value); - for (const key of Reflect.ownKeys(value)) { - const descriptor = Object.getOwnPropertyDescriptor(value, key); - if (!descriptor) return false; - if (typeof key === 'symbol') { - if (key === Symbol.toStringTag) return false; - if (descriptor.enumerable) return false; - continue; - } - if (!descriptor.enumerable) { - if (key === 'constructor' || key === 'then') return false; - continue; - } - if (key === '__proto__' || !('value' in descriptor)) return false; - if (!isPassivelyCloneable(descriptor.value, workflowGlobal, seen)) { - return false; - } + if (Array.isArray(value)) { + return isPassiveArray(value, workflowGlobal, seen); } - return true; + if (types.isMap(value)) return isPassiveMap(value, workflowGlobal, seen); + if (types.isSet(value)) return isPassiveSet(value, workflowGlobal, seen); + + const builtIn = isPassiveBuiltIn(value, workflowGlobal); + return builtIn ?? isPassivePlainObject(value, workflowGlobal, seen); } export function prepareRetainedStepInput( diff --git a/packages/core/src/runtime/suspension-handler.ts b/packages/core/src/runtime/suspension-handler.ts index e0c35f63ef..7fb976c9c0 100644 --- a/packages/core/src/runtime/suspension-handler.ts +++ b/packages/core/src/runtime/suspension-handler.ts @@ -131,6 +131,17 @@ export interface SuspensionHandlerResult { retainedStepInputsSafe: boolean; } +function getStepInput( + inputs: Map }>, + correlationId: string +): { value: unknown; global: Record } { + const input = inputs.get(correlationId); + if (!input) { + throw new Error(`Missing prepared input for step ${correlationId}`); + } + return input; +} + async function createHookEvent({ runId, hookEvent, @@ -559,12 +570,10 @@ export async function handleSuspension({ if (stepsNeedingCreation.has(queueItem.correlationId)) { ops.push( (async () => { - const input = inputsByCorrelationId.get(queueItem.correlationId); - if (!input) { - throw new Error( - `Missing prepared input for step ${queueItem.correlationId}` - ); - } + const input = getStepInput( + inputsByCorrelationId, + queueItem.correlationId + ); const dehydratedInput = await dehydrateStepArguments( input.value, runId, From 8d96711611d05a497d54968d1030021823b2ce0d Mon Sep 17 00:00:00 2001 From: Nathan Colosimo <110621881+NathanColosimo@users.noreply.github.com> Date: Fri, 17 Jul 2026 14:09:49 -0700 Subject: [PATCH 12/43] fix(core): guard proxied constructors in the passive-input walker constructorPrototype reads both realms' constructors via own-property descriptors only and refuses proxies before any descriptor read, so a proxied redefined global can never observe validation. --- .../src/runtime/retained-step-input.test.ts | 26 ++++++++++++++ .../core/src/runtime/retained-step-input.ts | 35 ++++++++++++++----- 2 files changed, 53 insertions(+), 8 deletions(-) diff --git a/packages/core/src/runtime/retained-step-input.test.ts b/packages/core/src/runtime/retained-step-input.test.ts index c316a42992..02c69e5410 100644 --- a/packages/core/src/runtime/retained-step-input.test.ts +++ b/packages/core/src/runtime/retained-step-input.test.ts @@ -185,4 +185,30 @@ describe('prepareRetainedStepInput review-hardening', () => { }); expect(vm.runInContext('globalThis.__retainedTestCalls', context)).toBe(0); }); + + it('never inspects a proxied workflow constructor', () => { + const { context, globalThis: workflowGlobal } = createContext({ + seed, + fixedTimestamp, + }); + const value = vm.runInContext( + `(() => { + const arr = [1]; + globalThis.__retainedTestCalls = 0; + globalThis.Array = new Proxy(function Array() {}, { + getOwnPropertyDescriptor(target, key) { + globalThis.__retainedTestCalls++; + return Reflect.getOwnPropertyDescriptor(target, key); + }, + }); + return arr; + })()`, + context + ); + + expect(prepareRetainedStepInput(value, workflowGlobal)).toEqual({ + retainable: false, + }); + expect(vm.runInContext('globalThis.__retainedTestCalls', context)).toBe(0); + }); }); diff --git a/packages/core/src/runtime/retained-step-input.ts b/packages/core/src/runtime/retained-step-input.ts index 129b1922be..5e83af11d7 100644 --- a/packages/core/src/runtime/retained-step-input.ts +++ b/packages/core/src/runtime/retained-step-input.ts @@ -27,20 +27,39 @@ function ownDataProperty(target: object, key: string): unknown { return descriptor && 'value' in descriptor ? descriptor.value : undefined; } +function constructorPrototype( + realmGlobal: Record, + constructorName: string +): object | undefined { + const ctor = ownDataProperty(realmGlobal, constructorName); + if ( + (typeof ctor !== 'function' && typeof ctor !== 'object') || + ctor === null || + types.isProxy(ctor) + ) { + return undefined; + } + const prototype = ownDataProperty(ctor, 'prototype'); + return typeof prototype === 'object' && + prototype !== null && + !types.isProxy(prototype) + ? prototype + : undefined; +} + function hasAllowedPrototype( value: object, workflowGlobal: Record, constructorName: string ): boolean { const prototype = Object.getPrototypeOf(value); - const hostConstructor = globalThis[ - constructorName as keyof typeof globalThis - ] as { prototype?: object } | undefined; - if (hostConstructor !== undefined && prototype === hostConstructor.prototype) - return true; - const vmConstructor = ownDataProperty(workflowGlobal, constructorName); - if (typeof vmConstructor !== 'function') return false; - return prototype === ownDataProperty(vmConstructor, 'prototype'); + return ( + prototype === + constructorPrototype( + globalThis as unknown as Record, + constructorName + ) || prototype === constructorPrototype(workflowGlobal, constructorName) + ); } function isArrayIndex(key: string): boolean { From fd73574c4b0ee3a53ebfaa0be73c93478910c73b Mon Sep 17 00:00:00 2001 From: Nathan Colosimo <110621881+NathanColosimo@users.noreply.github.com> Date: Fri, 17 Jul 2026 14:19:00 -0700 Subject: [PATCH 13/43] fix(core): preserve retention gate after rebase --- packages/core/src/runtime.ts | 23 +++++++++++++---------- 1 file changed, 13 insertions(+), 10 deletions(-) diff --git a/packages/core/src/runtime.ts b/packages/core/src/runtime.ts index 05f8f09795..fea67705c9 100644 --- a/packages/core/src/runtime.ts +++ b/packages/core/src/runtime.ts @@ -325,16 +325,19 @@ function canRetainWorkflowSession( suspension: WorkflowSuspension, events: Event[] ): boolean { - return ( - isVmRetentionEnabled() && - suspension.stepCount > 0 && - suspension.hookCount === 0 && - suspension.waitCount === 0 && - suspension.attributeCount === 0 && - suspension.hookDisposedCount === 0 && - suspension.abortCount === 0 && - !hasOpenHookOrWait(events) - ); + if ( + !isVmRetentionEnabled() || + suspension.stepCount === 0 || + suspension.hookCount > 0 || + suspension.waitCount > 0 || + suspension.attributeCount > 0 || + suspension.hookDisposedCount > 0 || + suspension.abortCount > 0 + ) { + return false; + } + const { openHook, openWait } = openHookAndWaitState(events); + return !openHook && !openWait; } /** From bd07e118d6278e6edb9a070e6b808e2d5888713e Mon Sep 17 00:00:00 2001 From: Nathan Colosimo <110621881+NathanColosimo@users.noreply.github.com> Date: Fri, 17 Jul 2026 14:23:16 -0700 Subject: [PATCH 14/43] fix(core): all-or-nothing clone batches; reject SAB views in digest - A mixed step batch (one unsafe sibling input) now serializes every input through the ordinary VM path: a clone snapshotted before an unsafe sibling's serialization runs its getters could otherwise durably capture stale sibling state. - crypto.subtle.digest rejects SharedArrayBuffer-backed views with TypeError, matching WebCrypto's BufferSource contract. --- packages/core/src/retained-vm-loop.test.ts | 22 ++++++ .../core/src/runtime/suspension-handler.ts | 70 ++++++++----------- packages/core/src/vm/index.test.ts | 13 ++++ packages/core/src/vm/index.ts | 7 ++ 4 files changed, 71 insertions(+), 41 deletions(-) diff --git a/packages/core/src/retained-vm-loop.test.ts b/packages/core/src/retained-vm-loop.test.ts index 532f981f79..d24b59bcc2 100644 --- a/packages/core/src/retained-vm-loop.test.ts +++ b/packages/core/src/retained-vm-loop.test.ts @@ -87,6 +87,19 @@ registerSerializationClass( RetainedSerializerValue ); +// A parallel batch where one sibling's input is unsafe must serialize the +// WHOLE batch through the ordinary VM path (all-or-nothing) and demote. +const mixedBatchWorkflow = `const s1 = globalThis[Symbol.for("WORKFLOW_USE_STEP")]("r_s1"); + const s2 = globalThis[Symbol.for("WORKFLOW_USE_STEP")]("r_s2"); + async function workflow() { + const [a, b] = await Promise.all([ + s1({ get x() { return 1; } }), + s2({ plain: true }), + ]); + return a + b; + } + globalThis.__private_workflows = new Map([["workflow", workflow]]);`; + // `crypto.subtle.digest` computes synchronously via node:crypto, so a // digest-using VM stays quiescent at suspension and remains retainable. const digestWorkflow = `const s1 = globalThis[Symbol.for("WORKFLOW_USE_STEP")]("r_s1"); @@ -297,6 +310,15 @@ describe('retained VM through the inline replay loop', () => { ).toBe(0); }); + it('demotes retention when any input in a parallel batch is unsafe', async () => { + const { vmBuilds, output } = await drive( + 'wrun_retained_mixed_batch', + mixedBatchWorkflow + ); + expect(output).toBeInstanceOf(Uint8Array); + expect(vmBuilds).toBeGreaterThan(1); + }); + it('retains a VM that used the synchronous crypto.subtle.digest', async () => { const { vmBuilds, output } = await drive( 'wrun_retained_digest', diff --git a/packages/core/src/runtime/suspension-handler.ts b/packages/core/src/runtime/suspension-handler.ts index 7fb976c9c0..ad980e1ce7 100644 --- a/packages/core/src/runtime/suspension-handler.ts +++ b/packages/core/src/runtime/suspension-handler.ts @@ -131,17 +131,6 @@ export interface SuspensionHandlerResult { retainedStepInputsSafe: boolean; } -function getStepInput( - inputs: Map }>, - correlationId: string -): { value: unknown; global: Record } { - const input = inputs.get(correlationId); - if (!input) { - throw new Error(`Missing prepared input for step ${correlationId}`); - } - return input; -} - async function createHookEvent({ runId, hookEvent, @@ -508,33 +497,31 @@ export async function handleSuspension({ // racing with concurrent handlers on step execution. const createdStepCorrelationIds = new Set(); + // All-or-nothing: clones are only used when EVERY new step input in the + // batch is passive. A mixed batch would let an unsafe sibling's + // serialization (which can run getters) mutate objects a clone already + // snapshotted, changing what the other step durably receives relative to + // the ordinary in-order VM traversal. let retainedStepInputsSafe = true; - const inputsByCorrelationId = new Map< - string, - { value: unknown; global: Record } - >(); - for (const queueItem of stepItems) { - if (!stepsNeedingCreation.has(queueItem.correlationId)) continue; - const value = { - args: queueItem.args, - closureVars: queueItem.closureVars, - thisVal: queueItem.thisVal, - }; - if (prepareForRetention) { - const prepared = prepareRetainedStepInput(value, suspension.globalThis); - if (prepared.retainable) { - inputsByCorrelationId.set(queueItem.correlationId, { - value: prepared.value, - global: globalThis, - }); - continue; + const clonesByCorrelationId = new Map(); + if (prepareForRetention) { + for (const queueItem of stepItems) { + if (!stepsNeedingCreation.has(queueItem.correlationId)) continue; + const prepared = prepareRetainedStepInput( + { + args: queueItem.args, + closureVars: queueItem.closureVars, + thisVal: queueItem.thisVal, + }, + suspension.globalThis + ); + if (!prepared.retainable) { + retainedStepInputsSafe = false; + clonesByCorrelationId.clear(); + break; } - retainedStepInputsSafe = false; + clonesByCorrelationId.set(queueItem.correlationId, prepared.value); } - inputsByCorrelationId.set(queueItem.correlationId, { - value, - global: suspension.globalThis, - }); } // Lazy inline start: defer the step_created write for up to @@ -570,15 +557,16 @@ export async function handleSuspension({ if (stepsNeedingCreation.has(queueItem.correlationId)) { ops.push( (async () => { - const input = getStepInput( - inputsByCorrelationId, - queueItem.correlationId - ); + const clone = clonesByCorrelationId.get(queueItem.correlationId); const dehydratedInput = await dehydrateStepArguments( - input.value, + clone ?? { + args: queueItem.args, + closureVars: queueItem.closureVars, + thisVal: queueItem.thisVal, + }, runId, encryptionKey, - input.global, + clone ? globalThis : suspension.globalThis, false, compression ); diff --git a/packages/core/src/vm/index.test.ts b/packages/core/src/vm/index.test.ts index 677a5ba1f1..4ff51153ae 100644 --- a/packages/core/src/vm/index.test.ts +++ b/packages/core/src/vm/index.test.ts @@ -370,3 +370,16 @@ describe('crypto.subtle.digest input validation', () => { ).rejects.toThrow(TypeError); }); }); + +describe('crypto.subtle.digest SharedArrayBuffer rejection', () => { + it('rejects SharedArrayBuffer-backed views like WebCrypto does', async () => { + const { context } = createContext({ seed, fixedTimestamp }); + + await expect( + vm.runInContext( + 'crypto.subtle.digest("SHA-256", new Uint8Array(new SharedArrayBuffer(4)))', + context + ) + ).rejects.toThrow(TypeError); + }); +}); diff --git a/packages/core/src/vm/index.ts b/packages/core/src/vm/index.ts index 03bc34d2d7..05874cc5c9 100644 --- a/packages/core/src/vm/index.ts +++ b/packages/core/src/vm/index.ts @@ -1,4 +1,5 @@ import { createHash } from 'node:crypto'; +import { types } from 'node:util'; import { runInContext, createContext as vmCreateContext } from 'node:vm'; import { WorkflowRuntimeError } from '@workflow/errors'; import seedrandom from 'seedrandom'; @@ -31,6 +32,12 @@ const arrayBufferByteLength = Object.getOwnPropertyDescriptor( // TypeError instead of allocating a Uint8Array of that length. function toDigestBytes(data: ArrayBuffer | ArrayBufferView): Uint8Array { if (ArrayBuffer.isView(data)) { + // WebCrypto's BufferSource excludes SharedArrayBuffer-backed views. + if (types.isSharedArrayBuffer(data.buffer)) { + throw new TypeError( + 'crypto.subtle.digest does not accept SharedArrayBuffer-backed views' + ); + } return new Uint8Array(data.buffer, data.byteOffset, data.byteLength); } arrayBufferByteLength.call(data); From fe425e6e9f98eef6be3447a606442e91abf44b43 Mon Sep 17 00:00:00 2001 From: Nathan Colosimo <110621881+NathanColosimo@users.noreply.github.com> Date: Fri, 17 Jul 2026 14:43:19 -0700 Subject: [PATCH 15/43] fix(core): narrow the fast path to prototype-independent types MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit devalue serializes Map/Set through the realm's iterator protocol and Date/RegExp/typed arrays through prototype getters, all of which workflow code can mutate — so their serialization is not provably passive and their bytes could differ between retained and cold modes. The fast path now accepts only primitives, plain objects, and plain arrays, which devalue traverses exclusively via own-property reads. Slot-bearing exotics decline even with a swapped prototype. The sandbox digest now reads view metadata (buffer/byteOffset/ byteLength) through captured intrinsic getters, so own properties shadowing them cannot change which bytes are hashed or bypass the SharedArrayBuffer rejection. --- .../docs/v5/configuration/runtime-tuning.mdx | 2 +- .../src/runtime/retained-step-input.test.ts | 49 ++++-- .../core/src/runtime/retained-step-input.ts | 141 +++++------------- packages/core/src/vm/index.test.ts | 21 +++ packages/core/src/vm/index.ts | 53 +++++-- 5 files changed, 136 insertions(+), 130 deletions(-) diff --git a/docs/content/docs/v5/configuration/runtime-tuning.mdx b/docs/content/docs/v5/configuration/runtime-tuning.mdx index 9c275ef8f0..f9ee1b7eb9 100644 --- a/docs/content/docs/v5/configuration/runtime-tuning.mdx +++ b/docs/content/docs/v5/configuration/runtime-tuning.mdx @@ -80,7 +80,7 @@ For example, a workflow can run a 10-minute inline step even with `WORKFLOW_REPL - Default: enabled - Keeps the suspended workflow VM alive across inline steps within one invocation, so each iteration of the inline loop appends only the newly written events instead of replaying the whole event log in a fresh VM. - Suspensions involving hooks, waits, or attributes — and any replay divergence — always fall back to a full replay. -- Step inputs containing accessors, proxies, functions, custom class serializers, or other values whose serialization could execute workflow code also fall back to replay. Plain data and standard in-memory collections remain retainable. +- Step inputs made of plain data (objects, arrays, and primitives) remain retainable. Anything whose serialization could execute or observe workflow code — accessors, proxies, functions, custom class serializers, and built-ins such as `Map`, `Set`, `Date`, or typed arrays — falls back to replay for that boundary. - Set `0` or `false` to replay from scratch in a fresh VM on every iteration. ### `WORKFLOW_INLINE_OWNERSHIP` diff --git a/packages/core/src/runtime/retained-step-input.test.ts b/packages/core/src/runtime/retained-step-input.test.ts index 02c69e5410..cc74ab54ab 100644 --- a/packages/core/src/runtime/retained-step-input.test.ts +++ b/packages/core/src/runtime/retained-step-input.test.ts @@ -14,14 +14,11 @@ describe('prepareRetainedStepInput', () => { fixedTimestamp, }); const value = vm.runInContext( - `(() => { - const buffer = new ArrayBuffer(8); - return { - nested: [{ ok: true }], - map: new Map([["key", new Set([1, 2])]]), - bytes: new Uint8Array(buffer, 2, 4), - }; - })()`, + `({ + nested: [{ ok: true }, "text", 42n], + sparse: [1, , 3], + flag: false, + })`, context ); @@ -30,9 +27,9 @@ describe('prepareRetainedStepInput', () => { expect(prepared.retainable).toBe(true); if (!prepared.retainable) return; expect(prepared.value).toEqual({ - nested: [{ ok: true }], - map: new Map([['key', new Set([1, 2])]]), - bytes: new Uint8Array(4), + nested: [{ ok: true }, 'text', 42n], + sparse: [1, undefined, 3], + flag: false, }); expect(Object.getPrototypeOf(prepared.value)).toBe(Object.prototype); }); @@ -45,10 +42,9 @@ describe('prepareRetainedStepInput', () => { const value = vm.runInContext( `({ nested: [{ ok: true, missing: undefined }], - map: new Map([["key", new Set([1, 2])]]), - bytes: new Uint8Array([1, 2, 3, 4]), - date: new Date(1234), - regexp: /workflow/gi, + sparse: [1, , 3], + big: 42n, + text: "workflow", })`, context ); @@ -68,6 +64,29 @@ describe('prepareRetainedStepInput', () => { expect(cloned).toEqual(original); }); + it('declines built-ins whose serialization consults realm prototypes', () => { + const { context, globalThis: workflowGlobal } = createContext({ + seed, + fixedTimestamp, + }); + for (const expression of [ + 'new Map([["k", 1]])', + 'new Set([1, 2])', + 'new Date(1234)', + '/workflow/gi', + 'new Uint8Array([1, 2])', + 'new ArrayBuffer(8)', + 'new DataView(new ArrayBuffer(8))', + 'new Error("boom")', + 'Object.assign(Object.create(Object.prototype), { m: new Map() })', + ]) { + const value = vm.runInContext(expression, context); + expect(prepareRetainedStepInput(value, workflowGlobal)).toEqual({ + retainable: false, + }); + } + }); + it('declines accessors without invoking them', () => { const { context, globalThis: workflowGlobal } = createContext({ seed, diff --git a/packages/core/src/runtime/retained-step-input.ts b/packages/core/src/runtime/retained-step-input.ts index 5e83af11d7..976c8ac611 100644 --- a/packages/core/src/runtime/retained-step-input.ts +++ b/packages/core/src/runtime/retained-step-input.ts @@ -4,21 +4,6 @@ export type RetainedStepInputPreparation = | { readonly retainable: true; readonly value: unknown } | { readonly retainable: false }; -const typedArrayNames = [ - 'BigInt64Array', - 'BigUint64Array', - 'Float16Array', - 'Float32Array', - 'Float64Array', - 'Int8Array', - 'Int16Array', - 'Int32Array', - 'Uint8Array', - 'Uint8ClampedArray', - 'Uint16Array', - 'Uint32Array', -] as const; - // Own data-property read that never performs a property Get — workflow code // can redefine its globals (or their `prototype` slots) with accessors, and // validation must not execute workflow-owned code. @@ -31,15 +16,15 @@ function constructorPrototype( realmGlobal: Record, constructorName: string ): object | undefined { - const ctor = ownDataProperty(realmGlobal, constructorName); + const constructor = ownDataProperty(realmGlobal, constructorName); if ( - (typeof ctor !== 'function' && typeof ctor !== 'object') || - ctor === null || - types.isProxy(ctor) + (typeof constructor !== 'function' && typeof constructor !== 'object') || + constructor === null || + types.isProxy(constructor) ) { return undefined; } - const prototype = ownDataProperty(ctor, 'prototype'); + const prototype = ownDataProperty(constructor, 'prototype'); return typeof prototype === 'object' && prototype !== null && !types.isProxy(prototype) @@ -62,6 +47,27 @@ function hasAllowedPrototype( ); } +// Cloneable exotics (per the structured-clone spec) whose devalue/reducer +// serialization reads prototype methods, getters, or iterators the workflow +// realm can mutate — serializing them is not provably passive, so they +// decline the fast path even when their prototype identity looks intact. +function isSlotBearingExotic(value: object): boolean { + return ( + types.isMap(value) || + types.isSet(value) || + types.isDate(value) || + types.isRegExp(value) || + types.isArrayBuffer(value) || + types.isSharedArrayBuffer(value) || + types.isDataView(value) || + types.isTypedArray(value) || + types.isBoxedPrimitive(value) || + types.isNativeError(value) || + types.isPromise(value) || + types.isArgumentsObject(value) + ); +} + function isArrayIndex(key: string): boolean { const index = Number(key); return ( @@ -107,78 +113,6 @@ function isPassiveArray( ); } -function isPassiveMap( - value: Map, - workflowGlobal: Record, - seen: WeakSet -): boolean { - if (!hasAllowedPrototype(value, workflowGlobal, 'Map')) return false; - if (Reflect.ownKeys(value).length > 0) return false; - let retainable = true; - Map.prototype.forEach.call(value, (entryValue: unknown, key: unknown) => { - retainable &&= - isPassivelyCloneable(key, workflowGlobal, seen) && - isPassivelyCloneable(entryValue, workflowGlobal, seen); - }); - return retainable; -} - -function isPassiveSet( - value: Set, - workflowGlobal: Record, - seen: WeakSet -): boolean { - if (!hasAllowedPrototype(value, workflowGlobal, 'Set')) return false; - if (Reflect.ownKeys(value).length > 0) return false; - let retainable = true; - Set.prototype.forEach.call(value, (entryValue: unknown) => { - retainable &&= isPassivelyCloneable(entryValue, workflowGlobal, seen); - }); - return retainable; -} - -function isPassiveBuiltIn( - value: object, - workflowGlobal: Record -): boolean | undefined { - if (types.isDate(value)) { - return ( - hasAllowedPrototype(value, workflowGlobal, 'Date') && - Reflect.ownKeys(value).length === 0 - ); - } - if (types.isRegExp(value)) { - return ( - hasAllowedPrototype(value, workflowGlobal, 'RegExp') && - Reflect.ownKeys(value).every((key) => key === 'lastIndex') - ); - } - if (types.isArrayBuffer(value)) { - return ( - hasAllowedPrototype(value, workflowGlobal, 'ArrayBuffer') && - Reflect.ownKeys(value).length === 0 - ); - } - if (types.isSharedArrayBuffer(value)) return false; - if (types.isDataView(value)) { - return ( - hasAllowedPrototype(value, workflowGlobal, 'DataView') && - Reflect.ownKeys(value).length === 0 - ); - } - if (types.isTypedArray(value)) { - return ( - Reflect.ownKeys(value).every( - (key) => typeof key === 'string' && isArrayIndex(key) - ) && - typedArrayNames.some((name) => - hasAllowedPrototype(value, workflowGlobal, name) - ) - ); - } - return undefined; -} - function isPassiveObjectProperty( object: object, key: string | symbol, @@ -212,14 +146,18 @@ function isPassivePlainObject( } /** - * Whether structured cloning `value` can avoid executing workflow-owned code. + * Whether structured cloning `value` is provably byte-equivalent to the + * ordinary VM serialization AND cannot execute workflow-owned code. * * The retained VM must not observe serialization side effects that a later - * cold replay cannot reconstruct. Proxy traps, accessors, functions, custom - * classes, and platform wrappers can all execute workflow code while devalue - * traverses them, so they deliberately decline the fast path. Plain data and - * standard in-memory collections are cloned into the host realm before - * serialization, keeping even mutable VM prototypes out of the write path. + * cold replay cannot reconstruct, and the durable bytes must not depend on + * `WORKFLOW_RETAINED_VM`. Only primitives, plain objects, and plain arrays + * qualify: devalue traverses them exclusively through own-property reads + * (`Object.keys`, `Object.hasOwn` + indices), so no workflow-realm prototype + * state can influence the output. Everything else — proxies, accessors, + * functions, custom classes, and built-ins like Map/Set/Date whose + * serialization consults mutable realm prototypes (iterators, getters) — + * declines the fast path and serializes through the ordinary VM traversal. */ function isPassivelyCloneable( value: unknown, @@ -243,11 +181,8 @@ function isPassivelyCloneable( if (Array.isArray(value)) { return isPassiveArray(value, workflowGlobal, seen); } - if (types.isMap(value)) return isPassiveMap(value, workflowGlobal, seen); - if (types.isSet(value)) return isPassiveSet(value, workflowGlobal, seen); - - const builtIn = isPassiveBuiltIn(value, workflowGlobal); - return builtIn ?? isPassivePlainObject(value, workflowGlobal, seen); + if (isSlotBearingExotic(value)) return false; + return isPassivePlainObject(value, workflowGlobal, seen); } export function prepareRetainedStepInput( diff --git a/packages/core/src/vm/index.test.ts b/packages/core/src/vm/index.test.ts index 4ff51153ae..7e778e10e1 100644 --- a/packages/core/src/vm/index.test.ts +++ b/packages/core/src/vm/index.test.ts @@ -383,3 +383,24 @@ describe('crypto.subtle.digest SharedArrayBuffer rejection', () => { ).rejects.toThrow(TypeError); }); }); + +describe('crypto.subtle.digest view metadata', () => { + it('reads view ranges from internal slots, ignoring shadowed properties', async () => { + const { context } = createContext({ seed, fixedTimestamp }); + + const result = await vm.runInContext( + `(() => { + const view = new Uint8Array([1, 2, 3, 4]); + Object.defineProperty(view, "byteLength", { value: 0 }); + Object.defineProperty(view, "byteOffset", { value: 2 }); + return crypto.subtle.digest("SHA-256", view); + })()`, + context + ); + const expected = await globalThis.crypto.subtle.digest( + 'SHA-256', + new Uint8Array([1, 2, 3, 4]) + ); + expect(new Uint8Array(result)).toEqual(new Uint8Array(expected)); + }); +}); diff --git a/packages/core/src/vm/index.ts b/packages/core/src/vm/index.ts index 05874cc5c9..39386d0378 100644 --- a/packages/core/src/vm/index.ts +++ b/packages/core/src/vm/index.ts @@ -20,28 +20,59 @@ const DIGEST_ALGORITHMS: Record = { 'SHA-512': 'sha512', }; -// biome-ignore lint/style/noNonNullAssertion: byteLength always exists on ArrayBuffer.prototype -const arrayBufferByteLength = Object.getOwnPropertyDescriptor( +// Intrinsic prototype getters, captured so view metadata is read from +// internal slots like WebCrypto's BufferSource conversion — own properties +// shadowing `buffer`/`byteOffset`/`byteLength` on a view must not change +// which bytes are hashed. +function intrinsicGetter(prototype: object, name: string) { + // biome-ignore lint/style/noNonNullAssertion: intrinsic accessors always exist + return Object.getOwnPropertyDescriptor(prototype, name)!.get!; +} +const arrayBufferByteLength = intrinsicGetter( ArrayBuffer.prototype, 'byteLength' -)!.get!; +); +const typedArrayPrototype = Object.getPrototypeOf( + Uint8Array.prototype +) as object; +const viewGetters = { + typedArray: { + buffer: intrinsicGetter(typedArrayPrototype, 'buffer'), + byteOffset: intrinsicGetter(typedArrayPrototype, 'byteOffset'), + byteLength: intrinsicGetter(typedArrayPrototype, 'byteLength'), + }, + dataView: { + buffer: intrinsicGetter(DataView.prototype, 'buffer'), + byteOffset: intrinsicGetter(DataView.prototype, 'byteOffset'), + byteLength: intrinsicGetter(DataView.prototype, 'byteLength'), + }, +}; -// WebCrypto BufferSource conversion: views pass through; anything else must -// be a real ArrayBuffer (the native byteLength getter is a brand check that -// works across vm realms) so that e.g. a plain number is rejected with -// TypeError instead of allocating a Uint8Array of that length. +// WebCrypto BufferSource conversion: typed-array/DataView views are read via +// internal slots; anything else must be a real ArrayBuffer (the native +// byteLength getter is a brand check that works across vm realms) so that +// e.g. a plain number is rejected with TypeError instead of allocating a +// Uint8Array of that length. function toDigestBytes(data: ArrayBuffer | ArrayBufferView): Uint8Array { - if (ArrayBuffer.isView(data)) { + if (types.isTypedArray(data) || types.isDataView(data)) { + const getters = types.isDataView(data) + ? viewGetters.dataView + : viewGetters.typedArray; + const buffer = getters.buffer.call(data) as ArrayBuffer; // WebCrypto's BufferSource excludes SharedArrayBuffer-backed views. - if (types.isSharedArrayBuffer(data.buffer)) { + if (types.isSharedArrayBuffer(buffer)) { throw new TypeError( 'crypto.subtle.digest does not accept SharedArrayBuffer-backed views' ); } - return new Uint8Array(data.buffer, data.byteOffset, data.byteLength); + return new Uint8Array( + buffer, + getters.byteOffset.call(data) as number, + getters.byteLength.call(data) as number + ); } arrayBufferByteLength.call(data); - return new Uint8Array(data); + return new Uint8Array(data as ArrayBuffer); } /** From 9a5f0705d2527b9b2b221e9223aed1790c353141 Mon Sep 17 00:00:00 2001 From: Nathan Colosimo <110621881+NathanColosimo@users.noreply.github.com> Date: Fri, 17 Jul 2026 15:08:10 -0700 Subject: [PATCH 16/43] fix(core): freeze serialization-consulted sandbox intrinsics MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit instanceof dispatch (Symbol.hasInstance via the constructor, Function.prototype, and Object.prototype), the class reducer's value.constructor walk, and devalue's Object/Array traversal all consult intrinsics workflow code could redefine — legally and deterministically — which would make the durable step input depend on WORKFLOW_RETAINED_VM (spoofed values serialize as e.g. Maps on the cold path but as plain clones on the retained path). Freeze Object/Array/Function (constructors and prototypes), the VM collection constructors, and every reducer-referenced global binding (absent ones pinned to undefined) right before the workflow bundle evaluates, so the retained-input equivalence holds by construction. Host-realm constructor escapes (e.g. TextEncoder.constructor) remain out of the determinism contract: code scheduling host timers was never deterministic under ordinary replay either; documented on canRetainWorkflowSession. --- .changeset/retain-workflow-vm.md | 2 +- .../v5/api-reference/workflow-globals.mdx | 1 + packages/core/src/runtime.ts | 6 ++ packages/core/src/vm/index.test.ts | 49 ++++++++++- packages/core/src/vm/index.ts | 83 +++++++++++++++++++ packages/core/src/workflow.ts | 6 +- 6 files changed, 144 insertions(+), 3 deletions(-) diff --git a/.changeset/retain-workflow-vm.md b/.changeset/retain-workflow-vm.md index 9949f21f78..e1853096a9 100644 --- a/.changeset/retain-workflow-vm.md +++ b/.changeset/retain-workflow-vm.md @@ -3,4 +3,4 @@ 'workflow': patch --- -Retain workflow execution across passive-data inline steps within one invocation; boundaries whose argument serialization could execute workflow code fall back to replay, and `WORKFLOW_RETAINED_VM=0` disables retention. `crypto.subtle.digest` in workflow functions now computes synchronously via `node:crypto`, so digest timing is deterministic under replay. `WeakRef`, `FinalizationRegistry`, `Atomics.waitAsync`, and async `WebAssembly` compilation are no longer exposed in workflow functions (GC observation and wall-clock timing cannot be replayed). +Retain workflow execution across passive-data inline steps within one invocation; boundaries whose argument serialization could execute workflow code fall back to replay, and `WORKFLOW_RETAINED_VM=0` disables retention. `crypto.subtle.digest` in workflow functions now computes synchronously via `node:crypto`, so digest timing is deterministic under replay. `WeakRef`, `FinalizationRegistry`, `Atomics.waitAsync`, and async `WebAssembly` compilation are no longer exposed in workflow functions (GC observation and wall-clock timing cannot be replayed), and the serialization-relevant intrinsics (`Object`/`Array`/`Function` and reducer-referenced global bindings) are frozen in the workflow sandbox. diff --git a/docs/content/docs/v5/api-reference/workflow-globals.mdx b/docs/content/docs/v5/api-reference/workflow-globals.mdx index 6faa7e3df6..59e8458b22 100644 --- a/docs/content/docs/v5/api-reference/workflow-globals.mdx +++ b/docs/content/docs/v5/api-reference/workflow-globals.mdx @@ -103,3 +103,4 @@ The following are **not available** in workflow functions. Move this logic to [s - **`WeakRef` and `FinalizationRegistry`**: garbage-collection timing is not deterministic, so observing it would make workflow code impossible to replay. (`WeakMap` and `WeakSet` remain available — they do not expose GC state.) - **`Atomics.waitAsync`**: a wall-clock timer, which cannot be replayed. Use [`sleep()`](/docs/api-reference/workflow/sleep) instead. - **Async `WebAssembly` compilation** (`compile`, `instantiate`, `compileStreaming`, `instantiateStreaming`): resolves on compile-thread timing. The synchronous `new WebAssembly.Module()` and `new WebAssembly.Instance()` constructors remain available. +- **Mutating core intrinsics**: `Object`, `Array`, and `Function` (constructors and prototypes), and the global bindings for serialization-relevant built-ins (`Map`, `Set`, `Date`, `RegExp`, typed arrays, `Headers`, `URL`, and similar) are frozen. Serialization dispatches on these, so redefining them would make replay unfaithful. diff --git a/packages/core/src/runtime.ts b/packages/core/src/runtime.ts index fea67705c9..02ce5b93ee 100644 --- a/packages/core/src/runtime.ts +++ b/packages/core/src/runtime.ts @@ -320,6 +320,12 @@ function openHookAndWaitState(events: Event[]): { * Retain only pure step boundaries with no out-of-band continuation source. * Attributes require replay; hooks and waits can wake another invocation. * `WORKFLOW_RETAINED_VM=0` disables retention entirely. + * + * Quiescence assumes workflow code stays inside the sandbox's determinism + * contract. Escaping to the host realm (e.g. recovering the host `Function` + * constructor from an exposed host class to schedule real timers) makes a + * workflow nondeterministic under ordinary replay too, and is not defended + * here. */ function canRetainWorkflowSession( suspension: WorkflowSuspension, diff --git a/packages/core/src/vm/index.test.ts b/packages/core/src/vm/index.test.ts index 7e778e10e1..92a5db593d 100644 --- a/packages/core/src/vm/index.test.ts +++ b/packages/core/src/vm/index.test.ts @@ -1,6 +1,6 @@ import * as vm from 'node:vm'; import { describe, expect, it } from 'vitest'; -import { createContext } from './index.js'; +import { createContext, freezeSerializationIntrinsics } from './index.js'; const seed = 'entropy seed'; const fixedTimestamp = 1234567890000; @@ -404,3 +404,50 @@ describe('crypto.subtle.digest view metadata', () => { expect(new Uint8Array(result)).toEqual(new Uint8Array(expected)); }); }); + +describe('freezeSerializationIntrinsics', () => { + it('pins serialization-consulted intrinsics against workflow mutation', () => { + const { context, globalThis: g } = createContext({ seed, fixedTimestamp }); + freezeSerializationIntrinsics(g); + + // Statics and prototypes reject strict-mode definition attempts. + expect(() => + vm.runInContext( + `"use strict"; Object.defineProperty(Map, Symbol.hasInstance, { value: () => true })`, + context + ) + ).toThrow(/not extensible/); + expect(() => + vm.runInContext(`"use strict"; Object.prototype.polluted = 1`, context) + ).toThrow(/not extensible/); + expect(() => + vm.runInContext( + `"use strict"; Function.prototype[Symbol.hasInstance] = () => true`, + context + ) + ).toThrow(/read only|not extensible/); + + // Bindings cannot be replaced, including intentionally absent ones. + vm.runInContext( + 'try { globalThis.Map = function () {} } catch {}', + context + ); + expect(vm.runInContext('typeof Map.prototype.get', context)).toBe( + 'function' + ); + vm.runInContext( + 'try { globalThis.Request = function () {} } catch {}', + context + ); + expect(vm.runInContext('typeof Request', context)).toBe('undefined'); + }); + + it('leaves ordinary workflow globals writable', () => { + const { context, globalThis: g } = createContext({ seed, fixedTimestamp }); + freezeSerializationIntrinsics(g); + + expect( + vm.runInContext('globalThis.myState = { ok: true }; myState.ok', context) + ).toBe(true); + }); +}); diff --git a/packages/core/src/vm/index.ts b/packages/core/src/vm/index.ts index 39386d0378..8963898f71 100644 --- a/packages/core/src/vm/index.ts +++ b/packages/core/src/vm/index.ts @@ -75,6 +75,89 @@ function toDigestBytes(data: ArrayBuffer | ArrayBufferView): Uint8Array { return new Uint8Array(data as ArrayBuffer); } +// Global bindings the serialization reducers dispatch on (`value instanceof +// global.X`) plus the intrinsics devalue's own traversal consults. Frozen so +// workflow code cannot make serialization classify a value differently +// between the retained-input fast path (host-realm clone) and the ordinary +// VM traversal — e.g. via a spoofed `Map[Symbol.hasInstance]` or a replaced +// global binding. +const SERIALIZATION_BINDINGS = [ + 'Object', + 'Array', + 'Function', + 'Map', + 'Set', + 'Date', + 'RegExp', + 'ArrayBuffer', + 'SharedArrayBuffer', + 'DataView', + 'Int8Array', + 'Uint8Array', + 'Uint8ClampedArray', + 'Int16Array', + 'Uint16Array', + 'Int32Array', + 'Uint32Array', + 'Float16Array', + 'Float32Array', + 'Float64Array', + 'BigInt64Array', + 'BigUint64Array', + 'Headers', + 'URL', + 'URLSearchParams', + 'DOMException', + 'AbortController', + 'AbortSignal', + 'Request', + 'Response', + 'ReadableStream', + 'WritableStream', + 'TransformStream', +] as const; + +/** + * Freeze the sandbox intrinsics that serialization consults. Called after + * ALL SDK globals are installed, immediately before the workflow bundle + * evaluates. + * + * `instanceof` dispatch resolves `Symbol.hasInstance` through the value's + * constructor, `Function.prototype`, and `Object.prototype`; the class + * reducer walks `value.constructor`; devalue reads `Object`/`Array` + * prototypes. Freezing these (and pinning every reducer-referenced global + * binding, including ones that are intentionally absent) makes the + * retained-input equivalence check in `retained-step-input.ts` hold by + * construction instead of by validation. + */ +export function freezeSerializationIntrinsics(g: typeof globalThis): void { + const vmIntrinsics = g as unknown as Record; + // VM-realm intrinsic objects: freeze the objects themselves so statics + // (e.g. `Map[Symbol.hasInstance]`) and prototype members cannot be added. + // Host-realm classes exposed into the sandbox are NOT frozen — only their + // bindings are pinned below. + Object.freeze(g.Object); + Object.freeze(g.Object.prototype); + Object.freeze(g.Array); + Object.freeze(g.Array.prototype); + Object.freeze(g.Function); + Object.freeze(g.Function.prototype); + for (const name of ['Map', 'Set', 'RegExp', 'ArrayBuffer', 'DataView']) { + Object.freeze(vmIntrinsics[name]); + } + for (const name of SERIALIZATION_BINDINGS) { + const descriptor = Object.getOwnPropertyDescriptor(g, name); + Object.defineProperty(g, name, { + // Absent bindings are pinned to undefined so workflow code cannot + // define a spoofing global the reducers would then dispatch on. + value: descriptor && 'value' in descriptor ? descriptor.value : undefined, + writable: false, + configurable: false, + enumerable: descriptor?.enumerable ?? false, + }); + } +} + /** * Creates a Node.js `vm.Context` configured to be usable for * executing workflow logic in a deterministic environment. diff --git a/packages/core/src/workflow.ts b/packages/core/src/workflow.ts index 83f25823b7..83a8154ca0 100644 --- a/packages/core/src/workflow.ts +++ b/packages/core/src/workflow.ts @@ -42,7 +42,7 @@ import { import * as Attribute from './telemetry/semantic-conventions.js'; import { applyWorkflowSuspensionToSpan, trace } from './telemetry.js'; import { getWorkflowRunStreamId } from './util.js'; -import { createContext } from './vm/index.js'; +import { createContext, freezeSerializationIntrinsics } from './vm/index.js'; import { runCachedWorkflowScript } from './vm/script-cache.js'; import { createAbortSignalStatics, @@ -1030,6 +1030,10 @@ function createWorkflowSession({ // Script rather than the line just past the end of the bundle. That path // is rare (it requires the lookup `?.get(...)` expression to throw) and // does not affect the workflow function or replay determinism. + // All SDK globals are installed; pin the serialization-consulted + // intrinsics before any workflow code can run. + freezeSerializationIntrinsics(vmGlobalThis); + runCachedWorkflowScript(workflowCode, filename, context); const workflowFn = runCachedWorkflowScript( `globalThis.__private_workflows?.get(${JSON.stringify(workflowRun.workflowName)})`, From 3bda890e8ddd836162e60645015a2b6548f54882 Mon Sep 17 00:00:00 2001 From: Nathan Colosimo <110621881+NathanColosimo@users.noreply.github.com> Date: Fri, 17 Jul 2026 15:21:08 -0700 Subject: [PATCH 17/43] fix(core): freeze every non-shared serialization constructor Typed-array constructors (and their shared %TypedArray% parent), the Date wrapper, and the session-local AbortController/AbortSignal/ Request/Response bindings were pinned but not frozen, so workflow code could still add Symbol.hasInstance statics that diverge reducer dispatch between the retained clone (host constructors) and ordinary VM serialization. Freeze every binding value that is not the shared host intrinsic; shared host objects are dispatched identically by both paths, so mutations there cannot cause mode divergence. --- packages/core/src/vm/index.test.ts | 12 ++++++++++++ packages/core/src/vm/index.ts | 28 +++++++++++++++++++--------- 2 files changed, 31 insertions(+), 9 deletions(-) diff --git a/packages/core/src/vm/index.test.ts b/packages/core/src/vm/index.test.ts index 92a5db593d..5896198788 100644 --- a/packages/core/src/vm/index.test.ts +++ b/packages/core/src/vm/index.test.ts @@ -426,6 +426,18 @@ describe('freezeSerializationIntrinsics', () => { context ) ).toThrow(/read only|not extensible/); + expect(() => + vm.runInContext( + `"use strict"; Object.defineProperty(Uint8Array, Symbol.hasInstance, { value: () => true })`, + context + ) + ).toThrow(/not extensible/); + expect(() => + vm.runInContext( + `"use strict"; Object.defineProperty(Date, Symbol.hasInstance, { value: () => true })`, + context + ) + ).toThrow(/not extensible/); // Bindings cannot be replaced, including intentionally absent ones. vm.runInContext( diff --git a/packages/core/src/vm/index.ts b/packages/core/src/vm/index.ts index 8963898f71..a6c94f09c8 100644 --- a/packages/core/src/vm/index.ts +++ b/packages/core/src/vm/index.ts @@ -131,26 +131,36 @@ const SERIALIZATION_BINDINGS = [ * construction instead of by validation. */ export function freezeSerializationIntrinsics(g: typeof globalThis): void { - const vmIntrinsics = g as unknown as Record; - // VM-realm intrinsic objects: freeze the objects themselves so statics - // (e.g. `Map[Symbol.hasInstance]`) and prototype members cannot be added. - // Host-realm classes exposed into the sandbox are NOT frozen — only their - // bindings are pinned below. Object.freeze(g.Object); Object.freeze(g.Object.prototype); Object.freeze(g.Array); Object.freeze(g.Array.prototype); Object.freeze(g.Function); Object.freeze(g.Function.prototype); - for (const name of ['Map', 'Set', 'RegExp', 'ArrayBuffer', 'DataView']) { - Object.freeze(vmIntrinsics[name]); - } + // The typed-array constructors dispatch `instanceof` through their shared + // %TypedArray% parent. + Object.freeze(Object.getPrototypeOf(g.Uint8Array)); + const hostGlobal = globalThis as unknown as Record; for (const name of SERIALIZATION_BINDINGS) { const descriptor = Object.getOwnPropertyDescriptor(g, name); + const value = + descriptor && 'value' in descriptor ? descriptor.value : undefined; + // Freeze VM-realm and session-local constructor objects so statics like + // `Symbol.hasInstance` cannot be added. A binding that references the + // shared host intrinsic stays unfrozen: both serialization paths + // dispatch through that same object, so a mutation cannot make them + // disagree (and freezing it would affect the whole process). + if ( + (typeof value === 'function' || + (typeof value === 'object' && value !== null)) && + value !== hostGlobal[name] + ) { + Object.freeze(value); + } Object.defineProperty(g, name, { // Absent bindings are pinned to undefined so workflow code cannot // define a spoofing global the reducers would then dispatch on. - value: descriptor && 'value' in descriptor ? descriptor.value : undefined, + value, writable: false, configurable: false, enumerable: descriptor?.enumerable ?? false, From 43788b8c69ce087af39cddd8fc1f367f122ac359 Mon Sep 17 00:00:00 2001 From: Nathan Colosimo <110621881+NathanColosimo@users.noreply.github.com> Date: Fri, 17 Jul 2026 15:40:00 -0700 Subject: [PATCH 18/43] fix(core): build retained clones in a pristine realm Replace structuredClone with an explicit deep copy into an SDK-private realm: clones previously inherited host prototypes, which workflow code can reach (e.g. via structuredClone's return values) and vandalize with Symbol.toStringTag or constructor overrides, shifting devalue's classification of the clone relative to the ordinary VM path. The pristine realm is unreachable by any user code, and the explicit copy serializes exactly what devalue traverses (own indices, own enumerable string props). Arrays also now decline own constructor properties, which the class reducer reads even when non-enumerable. --- .../src/runtime/retained-step-input.test.ts | 28 ++++++++- .../core/src/runtime/retained-step-input.ts | 62 +++++++++++++++++-- 2 files changed, 83 insertions(+), 7 deletions(-) diff --git a/packages/core/src/runtime/retained-step-input.test.ts b/packages/core/src/runtime/retained-step-input.test.ts index cc74ab54ab..0d151e9398 100644 --- a/packages/core/src/runtime/retained-step-input.test.ts +++ b/packages/core/src/runtime/retained-step-input.test.ts @@ -31,7 +31,11 @@ describe('prepareRetainedStepInput', () => { sparse: [1, undefined, 3], flag: false, }); - expect(Object.getPrototypeOf(prepared.value)).toBe(Object.prototype); + // The clone's prototype chain lives in a pristine realm: not the + // workflow realm, not the host realm. + const cloneProto = Object.getPrototypeOf(prepared.value); + expect(cloneProto).not.toBe(Object.prototype); + expect(Object.getPrototypeOf(cloneProto)).toBe(null); }); it('produces the same serialized bytes as ordinary VM traversal', async () => { @@ -64,6 +68,28 @@ describe('prepareRetainedStepInput', () => { expect(cloned).toEqual(original); }); + it('declines arrays with an own constructor property', () => { + const { context, globalThis: workflowGlobal } = createContext({ + seed, + fixedTimestamp, + }); + const value = vm.runInContext( + `(() => { + const arr = [1]; + Object.defineProperty(arr, "constructor", { + value: class Fake { static classId = "fake"; }, + enumerable: false, + }); + return arr; + })()`, + context + ); + + expect(prepareRetainedStepInput(value, workflowGlobal)).toEqual({ + retainable: false, + }); + }); + it('declines built-ins whose serialization consults realm prototypes', () => { const { context, globalThis: workflowGlobal } = createContext({ seed, diff --git a/packages/core/src/runtime/retained-step-input.ts b/packages/core/src/runtime/retained-step-input.ts index 976c8ac611..c8bff7a28a 100644 --- a/packages/core/src/runtime/retained-step-input.ts +++ b/packages/core/src/runtime/retained-step-input.ts @@ -1,9 +1,20 @@ import { types } from 'node:util'; +import { runInContext, createContext as vmCreateContext } from 'node:vm'; export type RetainedStepInputPreparation = | { readonly retainable: true; readonly value: unknown } | { readonly retainable: false }; +// A pristine realm nothing else can reach: clones are built from its Object +// and Array so their entire prototype chain is immune to intrinsic mutation +// in both the workflow realm and the host realm (workflow code can obtain +// host-realm objects through APIs like `structuredClone` and vandalize their +// prototypes). +const pristineRealm = runInContext( + '({ Object, Array })', + vmCreateContext() +) as { Object: ObjectConstructor; Array: ArrayConstructor }; + // Own data-property read that never performs a property Get — workflow code // can redefine its globals (or their `prototype` slots) with accessors, and // validation must not execute workflow-owned code. @@ -87,7 +98,11 @@ function isPassiveArrayProperty( if (key === 'length') return true; const descriptor = Object.getOwnPropertyDescriptor(array, key); if (!descriptor) return false; - if (key === 'then' || key === Symbol.toStringTag) return false; + // `then` and `constructor` are read by serialization dispatch even when + // non-enumerable (thenable assimilation, the class reducer). + if (key === 'then' || key === 'constructor' || key === Symbol.toStringTag) { + return false; + } if (typeof key !== 'string' || !isArrayIndex(key)) { return !descriptor.enumerable; } @@ -185,6 +200,42 @@ function isPassivelyCloneable( return isPassivePlainObject(value, workflowGlobal, seen); } +// Deep-copy walker-validated plain data into the pristine realm, copying +// exactly what devalue serializes: dense/sparse own indices for arrays and +// own enumerable string properties for objects. All reads are data-property +// reads — the walker already rejected everything else. +function cloneIntoPristineRealm( + value: unknown, + copies: WeakMap +): unknown { + if (value === null || typeof value !== 'object') return value; + const existing = copies.get(value); + if (existing !== undefined) return existing; + + if (Array.isArray(value)) { + const copy = new pristineRealm.Array(value.length) as unknown[]; + copies.set(value, copy); + for (const key of Reflect.ownKeys(value)) { + if (typeof key !== 'string' || !isArrayIndex(key)) continue; + copy[Number(key)] = cloneIntoPristineRealm( + (value as unknown as Record)[key], + copies + ); + } + return copy; + } + + const copy = new pristineRealm.Object() as Record; + copies.set(value, copy); + for (const key of Object.keys(value)) { + copy[key] = cloneIntoPristineRealm( + (value as Record)[key], + copies + ); + } + return copy; +} + export function prepareRetainedStepInput( value: unknown, workflowGlobal: Record @@ -192,9 +243,8 @@ export function prepareRetainedStepInput( if (!isPassivelyCloneable(value, workflowGlobal, new WeakSet())) { return { retainable: false }; } - try { - return { retainable: true, value: structuredClone(value) }; - } catch { - return { retainable: false }; - } + return { + retainable: true, + value: cloneIntoPristineRealm(value, new WeakMap()), + }; } From 2b736768446e9fd0f5ec0e03916e79f81ff57a0e Mon Sep 17 00:00:00 2001 From: Nathan Colosimo <110621881+NathanColosimo@users.noreply.github.com> Date: Fri, 17 Jul 2026 15:59:15 -0700 Subject: [PATCH 19/43] fix(core): verify host dispatch pristineness before retained cloning MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Host intrinsics are shared with the whole process and cannot be frozen, but workflow code can reach them (structuredClone results, exposed host classes) and install Symbol.hasInstance predicates that distinguish the original from its clone — or WORKFLOW_SERIALIZE statics on host Object/Array that the class reducer reads for host-prototype originals (hydrated step results). prepareRetainedStepInput now verifies, via own-descriptor reads only, that every host dispatch point is pristine and declines retention before any clone exists — so a spoofed predicate can never observe or capture a pristine-realm object. --- .../src/runtime/retained-step-input.test.ts | 30 +++++++ .../core/src/runtime/retained-step-input.ts | 84 ++++++++++++++++++- 2 files changed, 113 insertions(+), 1 deletion(-) diff --git a/packages/core/src/runtime/retained-step-input.test.ts b/packages/core/src/runtime/retained-step-input.test.ts index 0d151e9398..e4021e04a2 100644 --- a/packages/core/src/runtime/retained-step-input.test.ts +++ b/packages/core/src/runtime/retained-step-input.test.ts @@ -257,3 +257,33 @@ describe('prepareRetainedStepInput review-hardening', () => { expect(vm.runInContext('globalThis.__retainedTestCalls', context)).toBe(0); }); }); + +describe('host dispatch pristineness', () => { + it('declines retention while a host dispatch constructor is spoofed', () => { + const { context, globalThis: workflowGlobal } = createContext({ + seed, + fixedTimestamp, + }); + const value = vm.runInContext('({ plain: true })', context); + + expect(prepareRetainedStepInput(value, workflowGlobal).retainable).toBe( + true + ); + + Object.defineProperty(Headers, Symbol.hasInstance, { + value: () => false, + configurable: true, + }); + try { + expect(prepareRetainedStepInput(value, workflowGlobal)).toEqual({ + retainable: false, + }); + } finally { + delete (Headers as any)[Symbol.hasInstance]; + } + + expect(prepareRetainedStepInput(value, workflowGlobal).retainable).toBe( + true + ); + }); +}); diff --git a/packages/core/src/runtime/retained-step-input.ts b/packages/core/src/runtime/retained-step-input.ts index c8bff7a28a..d28b9be4a9 100644 --- a/packages/core/src/runtime/retained-step-input.ts +++ b/packages/core/src/runtime/retained-step-input.ts @@ -1,5 +1,6 @@ import { types } from 'node:util'; import { runInContext, createContext as vmCreateContext } from 'node:vm'; +import { WORKFLOW_SERIALIZE } from '@workflow/serde'; export type RetainedStepInputPreparation = | { readonly retainable: true; readonly value: unknown } @@ -15,6 +16,84 @@ const pristineRealm = runInContext( vmCreateContext() ) as { Object: ObjectConstructor; Array: ArrayConstructor }; +// Host constructors that serialization dispatches on when the clone is +// serialized under the host global — plus Object/Array, whose statics and +// prototypes the class reducer and devalue's tag lookup read for +// host-prototype originals (hydrated step results are host-realm objects). +// The workflow VM's own intrinsics are frozen (see vm/index.ts), but host +// intrinsics are shared with the whole process and cannot be frozen, so +// dispatch-pristineness is verified instead. Any dirt declines retention +// BEFORE a clone exists, so a spoofed predicate can never observe (or +// capture) a pristine-realm object. +const HOST_DISPATCH_CONSTRUCTORS = [ + 'Object', + 'Array', + 'Function', + 'Map', + 'Set', + 'Date', + 'RegExp', + 'ArrayBuffer', + 'SharedArrayBuffer', + 'DataView', + 'Int8Array', + 'Uint8Array', + 'Uint8ClampedArray', + 'Int16Array', + 'Uint16Array', + 'Int32Array', + 'Uint32Array', + 'Float16Array', + 'Float32Array', + 'Float64Array', + 'BigInt64Array', + 'BigUint64Array', + 'Headers', + 'URL', + 'URLSearchParams', + 'DOMException', + 'AbortController', + 'AbortSignal', + 'Request', + 'Response', + 'ReadableStream', + 'WritableStream', + 'TransformStream', +] as const; + +function hasOwn(target: object, key: string | symbol): boolean { + return Object.getOwnPropertyDescriptor(target, key) !== undefined; +} + +function isHostDispatchPristine(): boolean { + const hostGlobal = globalThis as unknown as Record; + for (const name of HOST_DISPATCH_CONSTRUCTORS) { + const constructor = hostGlobal[name]; + if (constructor === undefined) continue; + if (hasOwn(constructor as object, Symbol.hasInstance)) return false; + } + for (const constructor of [Object, Array]) { + if ( + hasOwn(constructor, WORKFLOW_SERIALIZE) || + hasOwn(constructor, 'classId') + ) { + return false; + } + } + for (const [prototype, constructor] of [ + [Object.prototype, Object], + [Array.prototype, Array], + ] as const) { + if ( + hasOwn(prototype, Symbol.toStringTag) || + ownDataProperty(prototype, 'constructor') !== constructor + ) { + return false; + } + } + return true; +} + // Own data-property read that never performs a property Get — workflow code // can redefine its globals (or their `prototype` slots) with accessors, and // validation must not execute workflow-owned code. @@ -240,7 +319,10 @@ export function prepareRetainedStepInput( value: unknown, workflowGlobal: Record ): RetainedStepInputPreparation { - if (!isPassivelyCloneable(value, workflowGlobal, new WeakSet())) { + if ( + !isHostDispatchPristine() || + !isPassivelyCloneable(value, workflowGlobal, new WeakSet()) + ) { return { retainable: false }; } return { From caf6a1a5673828a160880a95f56d774cb165ccab Mon Sep 17 00:00:00 2001 From: Nathan Colosimo <110621881+NathanColosimo@users.noreply.github.com> Date: Fri, 17 Jul 2026 16:16:30 -0700 Subject: [PATCH 20/43] fix(core): reject symbol properties from retained step inputs Reducers dispatch on symbol tags (e.g. the workflow abort-signal markers) that are non-enumerable and dropped by the pristine-realm copy, so a tagged object would serialize as an abort descriptor on the cold path but as plain data on the retained path. --- .../src/runtime/retained-step-input.test.ts | 24 +++++++++++++++++++ .../core/src/runtime/retained-step-input.ts | 15 ++++++------ 2 files changed, 32 insertions(+), 7 deletions(-) diff --git a/packages/core/src/runtime/retained-step-input.test.ts b/packages/core/src/runtime/retained-step-input.test.ts index e4021e04a2..4fe7a95646 100644 --- a/packages/core/src/runtime/retained-step-input.test.ts +++ b/packages/core/src/runtime/retained-step-input.test.ts @@ -287,3 +287,27 @@ describe('host dispatch pristineness', () => { ); }); }); + +describe('symbol-tagged inputs', () => { + it('declines objects carrying symbol properties (serialization dispatch tags)', () => { + const { context, globalThis: workflowGlobal } = createContext({ + seed, + fixedTimestamp, + }); + const value = vm.runInContext( + `(() => { + const signal = { aborted: false }; + Object.defineProperty(signal, Symbol.for("WORKFLOW_ABORT_STREAM_NAME"), { + value: "abort-stream", + enumerable: false, + }); + return { signal }; + })()`, + context + ); + + expect(prepareRetainedStepInput(value, workflowGlobal)).toEqual({ + retainable: false, + }); + }); +}); diff --git a/packages/core/src/runtime/retained-step-input.ts b/packages/core/src/runtime/retained-step-input.ts index d28b9be4a9..23ecf26bd2 100644 --- a/packages/core/src/runtime/retained-step-input.ts +++ b/packages/core/src/runtime/retained-step-input.ts @@ -177,12 +177,13 @@ function isPassiveArrayProperty( if (key === 'length') return true; const descriptor = Object.getOwnPropertyDescriptor(array, key); if (!descriptor) return false; - // `then` and `constructor` are read by serialization dispatch even when - // non-enumerable (thenable assimilation, the class reducer). - if (key === 'then' || key === 'constructor' || key === Symbol.toStringTag) { + // Symbol properties can be serialization dispatch tags (e.g. the workflow + // abort-signal markers) that the clone would drop; `then` and + // `constructor` are read by dispatch even when non-enumerable. + if (typeof key === 'symbol' || key === 'then' || key === 'constructor') { return false; } - if (typeof key !== 'string' || !isArrayIndex(key)) { + if (!isArrayIndex(key)) { return !descriptor.enumerable; } // Non-enumerable indices are dropped by structuredClone but persisted by @@ -215,9 +216,9 @@ function isPassiveObjectProperty( ): boolean { const descriptor = Object.getOwnPropertyDescriptor(object, key); if (!descriptor) return false; - if (typeof key === 'symbol') { - return key !== Symbol.toStringTag && !descriptor.enumerable; - } + // Symbol properties can be serialization dispatch tags (e.g. the workflow + // abort-signal markers) that the clone would drop. + if (typeof key === 'symbol') return false; if (!descriptor.enumerable) { return key !== 'constructor' && key !== 'then'; } From e961466c8c20e1901652235047a5488876d9cc75 Mon Sep 17 00:00:00 2001 From: Nathan Colosimo <110621881+NathanColosimo@users.noreply.github.com> Date: Fri, 17 Jul 2026 16:28:40 -0700 Subject: [PATCH 21/43] fix(core): retained inputs accept only own enumerable data properties MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Hidden own keys of any kind — non-enumerable properties, accessors, symbols — can be observed by serialization dispatch (reducer probes like .signal, thenable checks, the class reducer) while the pristine clone drops them. With no hidden own keys, every probe on an accepted object resolves deterministically through validated data or pristine prototypes. --- .../src/runtime/retained-step-input.test.ts | 24 +++++++++++++++ .../core/src/runtime/retained-step-input.ts | 29 ++++++++----------- 2 files changed, 36 insertions(+), 17 deletions(-) diff --git a/packages/core/src/runtime/retained-step-input.test.ts b/packages/core/src/runtime/retained-step-input.test.ts index 4fe7a95646..aaff8608ea 100644 --- a/packages/core/src/runtime/retained-step-input.test.ts +++ b/packages/core/src/runtime/retained-step-input.test.ts @@ -311,3 +311,27 @@ describe('symbol-tagged inputs', () => { }); }); }); + +describe('hidden properties', () => { + it('declines objects with non-enumerable properties (reducer probes read them)', () => { + const { context, globalThis: workflowGlobal } = createContext({ + seed, + fixedTimestamp, + }); + const value = vm.runInContext( + `(() => { + const controllerish = { plain: true }; + Object.defineProperty(controllerish, "signal", { + get() { return { aborted: false }; }, + enumerable: false, + }); + return controllerish; + })()`, + context + ); + + expect(prepareRetainedStepInput(value, workflowGlobal)).toEqual({ + retainable: false, + }); + }); +}); diff --git a/packages/core/src/runtime/retained-step-input.ts b/packages/core/src/runtime/retained-step-input.ts index 23ecf26bd2..9daf8105c5 100644 --- a/packages/core/src/runtime/retained-step-input.ts +++ b/packages/core/src/runtime/retained-step-input.ts @@ -177,18 +177,13 @@ function isPassiveArrayProperty( if (key === 'length') return true; const descriptor = Object.getOwnPropertyDescriptor(array, key); if (!descriptor) return false; - // Symbol properties can be serialization dispatch tags (e.g. the workflow - // abort-signal markers) that the clone would drop; `then` and - // `constructor` are read by dispatch even when non-enumerable. - if (typeof key === 'symbol' || key === 'then' || key === 'constructor') { - return false; - } - if (!isArrayIndex(key)) { - return !descriptor.enumerable; - } - // Non-enumerable indices are dropped by structuredClone but persisted by - // devalue, so they must decline the fast path. + // Only own enumerable data indices are passive. Anything hidden — symbol + // tags, non-enumerable properties, accessors — can be observed by + // serialization dispatch (reducer probes, thenable checks, the class + // reducer) while being dropped by the clone. return ( + typeof key === 'string' && + isArrayIndex(key) && descriptor.enumerable === true && 'value' in descriptor && isPassivelyCloneable(descriptor.value, workflowGlobal, seen) @@ -216,14 +211,14 @@ function isPassiveObjectProperty( ): boolean { const descriptor = Object.getOwnPropertyDescriptor(object, key); if (!descriptor) return false; - // Symbol properties can be serialization dispatch tags (e.g. the workflow - // abort-signal markers) that the clone would drop. - if (typeof key === 'symbol') return false; - if (!descriptor.enumerable) { - return key !== 'constructor' && key !== 'then'; - } + // Only own enumerable string-keyed data properties are passive. Anything + // hidden — symbol tags, non-enumerable properties, accessors — can be + // observed by serialization dispatch (reducer probes like `.signal`, + // thenable checks, the class reducer) while being dropped by the clone. return ( + typeof key === 'string' && key !== '__proto__' && + descriptor.enumerable === true && 'value' in descriptor && isPassivelyCloneable(descriptor.value, workflowGlobal, seen) ); From 3498706a6f8ddda48ba3273126d3a83f7382d35b Mon Sep 17 00:00:00 2001 From: Nathan Colosimo <110621881+NathanColosimo@users.noreply.github.com> Date: Fri, 17 Jul 2026 16:43:37 -0700 Subject: [PATCH 22/43] fix(core): freeze binding prototype chains for hasInstance lookup Symbol.hasInstance dispatch walks the constructor's prototype chain, so the frozen Date wrapper still exposed the unfrozen original VM Date it delegates statics to. Freeze each non-shared binding's full chain (stopping at host Function/Object prototypes) and verify host Object.prototype carries no added hasInstance on the detection side. --- .../core/src/runtime/retained-step-input.ts | 4 +++ packages/core/src/vm/index.test.ts | 8 ++++++ packages/core/src/vm/index.ts | 25 ++++++++++++++----- 3 files changed, 31 insertions(+), 6 deletions(-) diff --git a/packages/core/src/runtime/retained-step-input.ts b/packages/core/src/runtime/retained-step-input.ts index 9daf8105c5..5fb77b1362 100644 --- a/packages/core/src/runtime/retained-step-input.ts +++ b/packages/core/src/runtime/retained-step-input.ts @@ -91,6 +91,10 @@ function isHostDispatchPristine(): boolean { return false; } } + // Constructor prototype chains end at host Object.prototype, where an + // added @@hasInstance would be found by dispatch lookup. (Host + // Function.prototype's @@hasInstance is spec non-configurable.) + if (hasOwn(Object.prototype, Symbol.hasInstance)) return false; return true; } diff --git a/packages/core/src/vm/index.test.ts b/packages/core/src/vm/index.test.ts index 5896198788..e5127de31c 100644 --- a/packages/core/src/vm/index.test.ts +++ b/packages/core/src/vm/index.test.ts @@ -438,6 +438,14 @@ describe('freezeSerializationIntrinsics', () => { context ) ).toThrow(/not extensible/); + // @@hasInstance lookup walks the wrapper's prototype chain to the + // original VM Date, which must be frozen too. + expect(() => + vm.runInContext( + `"use strict"; Object.defineProperty(Object.getPrototypeOf(Date), Symbol.hasInstance, { value: () => true })`, + context + ) + ).toThrow(/not extensible/); // Bindings cannot be replaced, including intentionally absent ones. vm.runInContext( diff --git a/packages/core/src/vm/index.ts b/packages/core/src/vm/index.ts index a6c94f09c8..ef190f64e1 100644 --- a/packages/core/src/vm/index.ts +++ b/packages/core/src/vm/index.ts @@ -145,17 +145,30 @@ export function freezeSerializationIntrinsics(g: typeof globalThis): void { const descriptor = Object.getOwnPropertyDescriptor(g, name); const value = descriptor && 'value' in descriptor ? descriptor.value : undefined; - // Freeze VM-realm and session-local constructor objects so statics like - // `Symbol.hasInstance` cannot be added. A binding that references the - // shared host intrinsic stays unfrozen: both serialization paths - // dispatch through that same object, so a mutation cannot make them - // disagree (and freezing it would affect the whole process). + // Freeze VM-realm and session-local constructor objects — including + // their prototype chains, which `Symbol.hasInstance` lookup walks (the + // Date wrapper delegates statics to the unfrozen original VM Date via + // setPrototypeOf). A binding that references the shared host intrinsic + // stays unfrozen: both serialization paths dispatch through that same + // object, so a mutation cannot make them disagree (and freezing it + // would affect the whole process). Host Function.prototype's + // @@hasInstance is spec-immutable; host Object.prototype is verified + // separately (see isHostDispatchPristine). if ( (typeof value === 'function' || (typeof value === 'object' && value !== null)) && value !== hostGlobal[name] ) { - Object.freeze(value); + let chain: object | null = value as object; + while ( + chain !== null && + chain !== Function.prototype && + chain !== Object.prototype && + !Object.isFrozen(chain) + ) { + Object.freeze(chain); + chain = Object.getPrototypeOf(chain); + } } Object.defineProperty(g, name, { // Absent bindings are pinned to undefined so workflow code cannot From 59aaebb2dd464b9937e8fbdb4d4d52dddde15b29 Mon Sep 17 00:00:00 2001 From: Nathan Colosimo <110621881+NathanColosimo@users.noreply.github.com> Date: Tue, 21 Jul 2026 15:31:14 -0700 Subject: [PATCH 23/43] refactor(core): single-path retained serialization via pinned members (v2) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Serialize step inputs for retained boundaries through the one ordinary pipeline (original value, workflow global) instead of cloning into a pristine realm and serializing under the host global. With a single serialization event shared by every mode, durable bytes structurally cannot depend on WORKFLOW_RETAINED_VM; the only property retention needs is that serialization executes no workflow code, established by: - the passive walker (descriptor-only, unchanged in spirit), now also accepting Map/Set/Date/typed arrays/ArrayBuffer — the common built-in step arguments — via prototype-identity checks - vm/serialization-pins.ts: the 10 prototype members serialization executes for those built-ins (measured empirically), captured at context creation and identity-verified at each retained boundary; the 'touches only pinned members' test instruments every member and locks the list against serde drift - host-realm instances (hydrated step results) accepted without member verification: host members run host code, which cannot touch retained VM state Deletes the pristine clone realm, the host-dispatch pristineness checks, and the batch clone bookkeeping. --- .../docs/v5/configuration/runtime-tuning.mdx | 2 +- packages/core/src/retained-vm-loop.test.ts | 20 + .../src/runtime/retained-step-input.test.ts | 445 +++++++++--------- .../core/src/runtime/retained-step-input.ts | 348 +++++++------- .../core/src/runtime/suspension-handler.ts | 41 +- packages/core/src/vm/index.ts | 16 +- packages/core/src/vm/serialization-pins.ts | 115 +++++ 7 files changed, 551 insertions(+), 436 deletions(-) create mode 100644 packages/core/src/vm/serialization-pins.ts diff --git a/docs/content/docs/v5/configuration/runtime-tuning.mdx b/docs/content/docs/v5/configuration/runtime-tuning.mdx index 79389b4ed0..17322d7ca9 100644 --- a/docs/content/docs/v5/configuration/runtime-tuning.mdx +++ b/docs/content/docs/v5/configuration/runtime-tuning.mdx @@ -80,7 +80,7 @@ For example, a workflow can run a 10-minute inline step even with `WORKFLOW_REPL - Default: enabled - Keeps the suspended workflow VM alive across inline steps within one invocation, so each iteration of the inline loop appends only the newly written events instead of replaying the whole event log in a fresh VM. - Suspensions involving hooks, waits, or attributes — and any replay divergence — always fall back to a full replay. -- Step inputs made of plain data (objects, arrays, and primitives) remain retainable. Anything whose serialization could execute or observe workflow code — accessors, proxies, functions, custom class serializers, and built-ins such as `Map`, `Set`, `Date`, or typed arrays — falls back to replay for that boundary. +- Step inputs made of plain data (objects, arrays, primitives) and standard built-ins (`Map`, `Set`, `Date`, typed arrays, `ArrayBuffer`) remain retainable. Anything whose serialization could execute workflow code — accessors, proxies, functions, custom class serializers, `RegExp` — falls back to replay for that boundary, as does any input if the prototype members serialization relies on have been modified. - Set `0` or `false` to replay from scratch in a fresh VM on every iteration. ### `WORKFLOW_INLINE_OWNERSHIP` diff --git a/packages/core/src/retained-vm-loop.test.ts b/packages/core/src/retained-vm-loop.test.ts index d24b59bcc2..2085b1edad 100644 --- a/packages/core/src/retained-vm-loop.test.ts +++ b/packages/core/src/retained-vm-loop.test.ts @@ -100,6 +100,17 @@ const mixedBatchWorkflow = `const s1 = globalThis[Symbol.for("WORKFLOW_USE_STEP" } globalThis.__private_workflows = new Map([["workflow", workflow]]);`; +// Map/Date/typed-array arguments serialize through pinned prototype members +// (see vm/serialization-pins.ts), so these boundaries stay retainable. +const builtinArgsWorkflow = `const s1 = globalThis[Symbol.for("WORKFLOW_USE_STEP")]("r_s1"); + const s2 = globalThis[Symbol.for("WORKFLOW_USE_STEP")]("r_s2"); + async function workflow() { + const a = await s1({ index: new Map([["k", 1]]), when: new Date(1234) }); + const b = await s2(new Uint8Array([1, 2, 3])); + return a + b; + } + globalThis.__private_workflows = new Map([["workflow", workflow]]);`; + // `crypto.subtle.digest` computes synchronously via node:crypto, so a // digest-using VM stays quiescent at suspension and remains retainable. const digestWorkflow = `const s1 = globalThis[Symbol.for("WORKFLOW_USE_STEP")]("r_s1"); @@ -319,6 +330,15 @@ describe('retained VM through the inline replay loop', () => { expect(vmBuilds).toBeGreaterThan(1); }); + it('retains boundaries whose args are supported built-ins', async () => { + const { vmBuilds, output } = await drive( + 'wrun_retained_builtins', + builtinArgsWorkflow + ); + expect(output).toBeInstanceOf(Uint8Array); + expect(vmBuilds).toBe(1); + }); + it('retains a VM that used the synchronous crypto.subtle.digest', async () => { const { vmBuilds, output } = await drive( 'wrun_retained_digest', diff --git a/packages/core/src/runtime/retained-step-input.test.ts b/packages/core/src/runtime/retained-step-input.test.ts index aaff8608ea..43c0ab46df 100644 --- a/packages/core/src/runtime/retained-step-input.test.ts +++ b/packages/core/src/runtime/retained-step-input.test.ts @@ -1,18 +1,38 @@ import vm from 'node:vm'; import { describe, expect, it } from 'vitest'; -import * as stepSerialization from '../serialization/step.js'; +import { dehydrateStepArguments } from '../serialization.js'; import { createContext } from '../vm/index.js'; -import { prepareRetainedStepInput } from './retained-step-input.js'; +import { isRetainedSerializationPassive } from './retained-step-input.js'; const seed = 'retained-step-input'; const fixedTimestamp = 1_700_000_000_000; -describe('prepareRetainedStepInput', () => { - it('clones passive cross-realm data into the host realm', () => { - const { context, globalThis: workflowGlobal } = createContext({ - seed, - fixedTimestamp, - }); +function makeContext() { + const { context, globalThis: workflowGlobal } = createContext({ + seed, + fixedTimestamp, + }); + // Mimic the globals workflow.ts installs before any serialization happens + // (the stream/request reducers dispatch on them unguarded). + for (const name of [ + 'ReadableStream', + 'WritableStream', + 'TransformStream', + 'Request', + 'Response', + 'AbortController', + 'AbortSignal', + ]) { + if ((workflowGlobal as any)[name] === undefined) { + (workflowGlobal as any)[name] = (globalThis as any)[name]; + } + } + return { context, workflowGlobal }; +} + +describe('isRetainedSerializationPassive', () => { + it('accepts plain cross-realm data', () => { + const { context, workflowGlobal } = makeContext(); const value = vm.runInContext( `({ nested: [{ ok: true }, "text", 42n], @@ -22,102 +42,55 @@ describe('prepareRetainedStepInput', () => { context ); - const prepared = prepareRetainedStepInput(value, workflowGlobal); - - expect(prepared.retainable).toBe(true); - if (!prepared.retainable) return; - expect(prepared.value).toEqual({ - nested: [{ ok: true }, 'text', 42n], - sparse: [1, undefined, 3], - flag: false, - }); - // The clone's prototype chain lives in a pristine realm: not the - // workflow realm, not the host realm. - const cloneProto = Object.getPrototypeOf(prepared.value); - expect(cloneProto).not.toBe(Object.prototype); - expect(Object.getPrototypeOf(cloneProto)).toBe(null); + expect(isRetainedSerializationPassive(value, workflowGlobal)).toBe(true); }); - it('produces the same serialized bytes as ordinary VM traversal', async () => { - const { context, globalThis: workflowGlobal } = createContext({ - seed, - fixedTimestamp, - }); - const value = vm.runInContext( - `({ - nested: [{ ok: true, missing: undefined }], - sparse: [1, , 3], - big: 42n, - text: "workflow", - })`, - context - ); - const prepared = prepareRetainedStepInput(value, workflowGlobal); - expect(prepared.retainable).toBe(true); - if (!prepared.retainable) return; - - const original = await stepSerialization.serialize(value, undefined, { - global: workflowGlobal, - }); - const cloned = await stepSerialization.serialize( - prepared.value, - undefined, - { global: globalThis } - ); - - expect(cloned).toEqual(original); + it('accepts the supported built-ins with pristine pins', () => { + const { context, workflowGlobal } = makeContext(); + for (const expression of [ + 'new Map([["k", { ok: true }]])', + 'new Set([1, "two"])', + 'new Date(1234)', + 'new Uint8Array([1, 2, 3])', + 'new Float32Array([1.5])', + 'new ArrayBuffer(8)', + '({ when: new Date(0), bytes: new Uint8Array(2), index: new Map() })', + ]) { + const value = vm.runInContext(expression, context); + expect(isRetainedSerializationPassive(value, workflowGlobal)).toBe(true); + } }); - it('declines arrays with an own constructor property', () => { - const { context, globalThis: workflowGlobal } = createContext({ - seed, - fixedTimestamp, - }); - const value = vm.runInContext( - `(() => { - const arr = [1]; - Object.defineProperty(arr, "constructor", { - value: class Fake { static classId = "fake"; }, - enumerable: false, - }); - return arr; - })()`, - context - ); + it('accepts host-realm instances (hydrated step results)', () => { + const { workflowGlobal } = makeContext(); + const value = { + map: new Map([['k', 1]]), + date: new Date(1234), + bytes: new Uint8Array([9]), + }; - expect(prepareRetainedStepInput(value, workflowGlobal)).toEqual({ - retainable: false, - }); + expect(isRetainedSerializationPassive(value, workflowGlobal)).toBe(true); }); - it('declines built-ins whose serialization consults realm prototypes', () => { - const { context, globalThis: workflowGlobal } = createContext({ - seed, - fixedTimestamp, - }); + it('declines types whose serialization is not pinned', () => { + const { context, workflowGlobal } = makeContext(); for (const expression of [ - 'new Map([["k", 1]])', - 'new Set([1, 2])', - 'new Date(1234)', '/workflow/gi', - 'new Uint8Array([1, 2])', - 'new ArrayBuffer(8)', 'new DataView(new ArrayBuffer(8))', + 'new SharedArrayBuffer(8)', + 'new Uint8Array(new SharedArrayBuffer(4))', 'new Error("boom")', - 'Object.assign(Object.create(Object.prototype), { m: new Map() })', + 'new (class Sub extends Map {})()', + 'Object.assign(new Map(), { expando: 1 })', + 'Object.create(null)', ]) { const value = vm.runInContext(expression, context); - expect(prepareRetainedStepInput(value, workflowGlobal)).toEqual({ - retainable: false, - }); + expect(isRetainedSerializationPassive(value, workflowGlobal)).toBe(false); } }); it('declines accessors without invoking them', () => { - const { context, globalThis: workflowGlobal } = createContext({ - seed, - fixedTimestamp, - }); + const { context, workflowGlobal } = makeContext(); const value = vm.runInContext( `(() => { globalThis.__retainedTestCalls = 0; @@ -126,17 +99,12 @@ describe('prepareRetainedStepInput', () => { context ); - expect(prepareRetainedStepInput(value, workflowGlobal)).toEqual({ - retainable: false, - }); + expect(isRetainedSerializationPassive(value, workflowGlobal)).toBe(false); expect(vm.runInContext('globalThis.__retainedTestCalls', context)).toBe(0); }); it('declines proxies without invoking their traps', () => { - const { context, globalThis: workflowGlobal } = createContext({ - seed, - fixedTimestamp, - }); + const { context, workflowGlobal } = makeContext(); const value = vm.runInContext( `(() => { globalThis.__retainedTestCalls = 0; @@ -150,17 +118,12 @@ describe('prepareRetainedStepInput', () => { context ); - expect(prepareRetainedStepInput(value, workflowGlobal)).toEqual({ - retainable: false, - }); + expect(isRetainedSerializationPassive(value, workflowGlobal)).toBe(false); expect(vm.runInContext('globalThis.__retainedTestCalls', context)).toBe(0); }); it('declines custom class serializers without invoking them', () => { - const { context, globalThis: workflowGlobal } = createContext({ - seed, - fixedTimestamp, - }); + const { context, workflowGlobal } = makeContext(); const value = vm.runInContext( `(() => { globalThis.__retainedTestCalls = 0; @@ -177,161 +140,197 @@ describe('prepareRetainedStepInput', () => { context ); - expect(prepareRetainedStepInput(value, workflowGlobal)).toEqual({ - retainable: false, - }); + expect(isRetainedSerializationPassive(value, workflowGlobal)).toBe(false); expect(vm.runInContext('globalThis.__retainedTestCalls', context)).toBe(0); }); -}); -describe('prepareRetainedStepInput review-hardening', () => { - it('declines non-enumerable array indices (structuredClone would drop them)', () => { - const { context, globalThis: workflowGlobal } = createContext({ - seed, - fixedTimestamp, - }); - const value = vm.runInContext( + it('declines hidden own keys (symbols, non-enumerables, constructor)', () => { + const { context, workflowGlobal } = makeContext(); + for (const expression of [ + `(() => { + const tagged = { plain: true }; + Object.defineProperty(tagged, Symbol.for("WORKFLOW_ABORT_STREAM_NAME"), { + value: "abort-stream", enumerable: false, + }); + return tagged; + })()`, + `(() => { + const hidden = { plain: true }; + Object.defineProperty(hidden, "signal", { + get() { return { aborted: false }; }, enumerable: false, + }); + return hidden; + })()`, + `(() => { + const arr = [1]; + Object.defineProperty(arr, "constructor", { + value: class Fake { static classId = "fake"; }, enumerable: false, + }); + return arr; + })()`, `(() => { const arr = [1, 2]; Object.defineProperty(arr, "0", { value: 7, enumerable: false }); return arr; })()`, + ]) { + const value = vm.runInContext(expression, context); + expect(isRetainedSerializationPassive(value, workflowGlobal)).toBe(false); + } + }); + + it('declines everything once a pinned member is patched', () => { + const { context, workflowGlobal } = makeContext(); + const plain = vm.runInContext('({ ok: true })', context); + expect(isRetainedSerializationPassive(plain, workflowGlobal)).toBe(true); + + vm.runInContext( + `globalThis.__origIterator = Map.prototype[Symbol.iterator]; + Map.prototype[Symbol.iterator] = function () { return globalThis.__origIterator.call(this); };`, context ); + expect(isRetainedSerializationPassive(plain, workflowGlobal)).toBe(false); - expect(prepareRetainedStepInput(value, workflowGlobal)).toEqual({ - retainable: false, - }); - }); - - it('never performs a property Get on redefined workflow globals', () => { - const { context, globalThis: workflowGlobal } = createContext({ - seed, - fixedTimestamp, - }); - const value = vm.runInContext( - `(() => { - globalThis.__retainedTestCalls = 0; - const arr = [1]; - for (const name of ["Array", "Object"]) { - Object.defineProperty(globalThis, name, { - get() { globalThis.__retainedTestCalls++; throw new Error("boom"); }, - }); - } - return { arr }; - })()`, + vm.runInContext( + 'Map.prototype[Symbol.iterator] = globalThis.__origIterator;', context ); + expect(isRetainedSerializationPassive(plain, workflowGlobal)).toBe(true); + }); - // The vandalized globals make the VM-realm prototypes unverifiable, so - // validation declines — without throwing or invoking the getters. - expect(prepareRetainedStepInput(value, workflowGlobal)).toEqual({ - retainable: false, - }); - expect(vm.runInContext('globalThis.__retainedTestCalls', context)).toBe(0); + it('declines per pinned member: iterator next, Date methods', () => { + for (const patch of [ + 'Object.getPrototypeOf((new Map())[Symbol.iterator]()).next = function () {};', + 'Object.getPrototypeOf((new Set())[Symbol.iterator]()).next = function () {};', + 'Date.prototype.toISOString = function () { return "spoofed"; };', + 'Date.prototype.getDate = function () { return 1; };', + ]) { + const { context, workflowGlobal } = makeContext(); + const plain = vm.runInContext('({ ok: true })', context); + vm.runInContext(patch, context); + expect(isRetainedSerializationPassive(plain, workflowGlobal)).toBe(false); + } }); - it('never inspects a proxied workflow constructor', () => { - const { context, globalThis: workflowGlobal } = createContext({ - seed, - fixedTimestamp, - }); + it('declines typed arrays whose subclass prototype shadows a pinned getter', () => { + const { context, workflowGlobal } = makeContext(); const value = vm.runInContext( `(() => { - const arr = [1]; - globalThis.__retainedTestCalls = 0; - globalThis.Array = new Proxy(function Array() {}, { - getOwnPropertyDescriptor(target, key) { - globalThis.__retainedTestCalls++; - return Reflect.getOwnPropertyDescriptor(target, key); - }, + Object.defineProperty(Float32Array.prototype, "buffer", { + get() { return new ArrayBuffer(0); }, configurable: true, }); - return arr; + return new Float32Array([1.5]); })()`, context ); - expect(prepareRetainedStepInput(value, workflowGlobal)).toEqual({ - retainable: false, - }); - expect(vm.runInContext('globalThis.__retainedTestCalls', context)).toBe(0); + expect(isRetainedSerializationPassive(value, workflowGlobal)).toBe(false); + // Pins themselves are intact, so unrelated values stay retainable. + const plain = vm.runInContext('({ ok: true })', context); + expect(isRetainedSerializationPassive(plain, workflowGlobal)).toBe(true); }); }); -describe('host dispatch pristineness', () => { - it('declines retention while a host dispatch constructor is spoofed', () => { - const { context, globalThis: workflowGlobal } = createContext({ - seed, - fixedTimestamp, - }); - const value = vm.runInContext('({ plain: true })', context); +describe('serialization touches only pinned members', () => { + // THE coupling test: the pin list in vm/serialization-pins.ts is only sound + // if it covers every prototype member `dehydrateStepArguments` executes for + // the supported built-ins. Wrap every configurable member on the relevant + // prototypes with a recorder and assert the serializer hits nothing beyond + // the pinned set. If serde changes what it touches, this fails loudly — + // update the pin list (and this list) together. + const PINNED = new Set([ + 'Map.prototype.Symbol(Symbol.iterator)', + '%MapIteratorPrototype%.next', + 'Set.prototype.Symbol(Symbol.iterator)', + '%SetIteratorPrototype%.next', + 'Date.prototype.getDate', + 'Date.prototype.toISOString', + '%TypedArray%.prototype.buffer', + '%TypedArray%.prototype.byteOffset', + '%TypedArray%.prototype.byteLength', + 'ArrayBuffer.prototype.byteLength', + ]); - expect(prepareRetainedStepInput(value, workflowGlobal).retainable).toBe( - true - ); - - Object.defineProperty(Headers, Symbol.hasInstance, { - value: () => false, - configurable: true, - }); - try { - expect(prepareRetainedStepInput(value, workflowGlobal)).toEqual({ - retainable: false, - }); - } finally { - delete (Headers as any)[Symbol.hasInstance]; - } + it('for Map, Set, Date, typed arrays, and ArrayBuffer', async () => { + const { context, workflowGlobal } = makeContext(); + const g = workflowGlobal as any; + const touched = new Set(); - expect(prepareRetainedStepInput(value, workflowGlobal).retainable).toBe( - true - ); - }); -}); + const wrapPrototype = (prototype: object, label: string) => { + for (const key of Reflect.ownKeys(prototype)) { + if (key === 'constructor') continue; + const descriptor = Object.getOwnPropertyDescriptor(prototype, key); + if (!descriptor || !descriptor.configurable) continue; + const name = `${label}.${String(key)}`; + if (typeof descriptor.value === 'function') { + const original = descriptor.value; + Object.defineProperty(prototype, key, { + ...descriptor, + value: function (this: unknown, ...args: unknown[]) { + touched.add(name); + return original.apply(this, args); + }, + }); + } else if (descriptor.get) { + const originalGet = descriptor.get; + Object.defineProperty(prototype, key, { + ...descriptor, + get() { + touched.add(name); + return originalGet.call(this); + }, + set: descriptor.set, + }); + } + } + }; -describe('symbol-tagged inputs', () => { - it('declines objects carrying symbol properties (serialization dispatch tags)', () => { - const { context, globalThis: workflowGlobal } = createContext({ - seed, - fixedTimestamp, - }); - const value = vm.runInContext( - `(() => { - const signal = { aborted: false }; - Object.defineProperty(signal, Symbol.for("WORKFLOW_ABORT_STREAM_NAME"), { - value: "abort-stream", - enumerable: false, - }); - return { signal }; - })()`, + const values = vm.runInContext( + `({ + map: new Map([["k", 1]]), + set: new Set([1, 2]), + date: new Date(1234), + f32: new Float32Array([1.5, 2.5]), + u8: new Uint8Array([1, 2, 3]), + ab: new ArrayBuffer(8), + })`, context ); - expect(prepareRetainedStepInput(value, workflowGlobal)).toEqual({ - retainable: false, - }); - }); -}); - -describe('hidden properties', () => { - it('declines objects with non-enumerable properties (reducer probes read them)', () => { - const { context, globalThis: workflowGlobal } = createContext({ - seed, - fixedTimestamp, - }); - const value = vm.runInContext( - `(() => { - const controllerish = { plain: true }; - Object.defineProperty(controllerish, "signal", { - get() { return { aborted: false }; }, - enumerable: false, - }); - return controllerish; - })()`, - context + wrapPrototype(g.Map.prototype, 'Map.prototype'); + wrapPrototype( + Object.getPrototypeOf(new g.Map()[Symbol.iterator]()), + '%MapIteratorPrototype%' + ); + wrapPrototype(g.Set.prototype, 'Set.prototype'); + wrapPrototype( + Object.getPrototypeOf(new g.Set()[Symbol.iterator]()), + '%SetIteratorPrototype%' ); + const datePrototype = Object.getOwnPropertyDescriptor(g.Date, 'prototype')! + .value as object; + wrapPrototype(datePrototype, 'Date.prototype'); + wrapPrototype( + Object.getPrototypeOf(g.Uint8Array.prototype), + '%TypedArray%.prototype' + ); + wrapPrototype(g.Uint8Array.prototype, 'Uint8Array.prototype'); + wrapPrototype(g.Float32Array.prototype, 'Float32Array.prototype'); + wrapPrototype(g.ArrayBuffer.prototype, 'ArrayBuffer.prototype'); - expect(prepareRetainedStepInput(value, workflowGlobal)).toEqual({ - retainable: false, - }); + for (const value of Object.values(values as Record)) { + touched.clear(); + await dehydrateStepArguments( + { args: [value], closureVars: undefined, thisVal: undefined }, + 'wrun_pin_coverage', + undefined, + g, + false, + false + ); + for (const name of touched) { + expect(PINNED, `unpinned member executed: ${name}`).toContain(name); + } + } }); }); diff --git a/packages/core/src/runtime/retained-step-input.ts b/packages/core/src/runtime/retained-step-input.ts index 5fb77b1362..0e478cd8a2 100644 --- a/packages/core/src/runtime/retained-step-input.ts +++ b/packages/core/src/runtime/retained-step-input.ts @@ -1,102 +1,8 @@ import { types } from 'node:util'; -import { runInContext, createContext as vmCreateContext } from 'node:vm'; -import { WORKFLOW_SERIALIZE } from '@workflow/serde'; - -export type RetainedStepInputPreparation = - | { readonly retainable: true; readonly value: unknown } - | { readonly retainable: false }; - -// A pristine realm nothing else can reach: clones are built from its Object -// and Array so their entire prototype chain is immune to intrinsic mutation -// in both the workflow realm and the host realm (workflow code can obtain -// host-realm objects through APIs like `structuredClone` and vandalize their -// prototypes). -const pristineRealm = runInContext( - '({ Object, Array })', - vmCreateContext() -) as { Object: ObjectConstructor; Array: ArrayConstructor }; - -// Host constructors that serialization dispatches on when the clone is -// serialized under the host global — plus Object/Array, whose statics and -// prototypes the class reducer and devalue's tag lookup read for -// host-prototype originals (hydrated step results are host-realm objects). -// The workflow VM's own intrinsics are frozen (see vm/index.ts), but host -// intrinsics are shared with the whole process and cannot be frozen, so -// dispatch-pristineness is verified instead. Any dirt declines retention -// BEFORE a clone exists, so a spoofed predicate can never observe (or -// capture) a pristine-realm object. -const HOST_DISPATCH_CONSTRUCTORS = [ - 'Object', - 'Array', - 'Function', - 'Map', - 'Set', - 'Date', - 'RegExp', - 'ArrayBuffer', - 'SharedArrayBuffer', - 'DataView', - 'Int8Array', - 'Uint8Array', - 'Uint8ClampedArray', - 'Int16Array', - 'Uint16Array', - 'Int32Array', - 'Uint32Array', - 'Float16Array', - 'Float32Array', - 'Float64Array', - 'BigInt64Array', - 'BigUint64Array', - 'Headers', - 'URL', - 'URLSearchParams', - 'DOMException', - 'AbortController', - 'AbortSignal', - 'Request', - 'Response', - 'ReadableStream', - 'WritableStream', - 'TransformStream', -] as const; - -function hasOwn(target: object, key: string | symbol): boolean { - return Object.getOwnPropertyDescriptor(target, key) !== undefined; -} - -function isHostDispatchPristine(): boolean { - const hostGlobal = globalThis as unknown as Record; - for (const name of HOST_DISPATCH_CONSTRUCTORS) { - const constructor = hostGlobal[name]; - if (constructor === undefined) continue; - if (hasOwn(constructor as object, Symbol.hasInstance)) return false; - } - for (const constructor of [Object, Array]) { - if ( - hasOwn(constructor, WORKFLOW_SERIALIZE) || - hasOwn(constructor, 'classId') - ) { - return false; - } - } - for (const [prototype, constructor] of [ - [Object.prototype, Object], - [Array.prototype, Array], - ] as const) { - if ( - hasOwn(prototype, Symbol.toStringTag) || - ownDataProperty(prototype, 'constructor') !== constructor - ) { - return false; - } - } - // Constructor prototype chains end at host Object.prototype, where an - // added @@hasInstance would be found by dispatch lookup. (Host - // Function.prototype's @@hasInstance is spec non-configurable.) - if (hasOwn(Object.prototype, Symbol.hasInstance)) return false; - return true; -} +import { + getSerializationPins, + verifySerializationPins, +} from '../vm/serialization-pins.js'; // Own data-property read that never performs a property Get — workflow code // can redefine its globals (or their `prototype` slots) with accessors, and @@ -141,27 +47,6 @@ function hasAllowedPrototype( ); } -// Cloneable exotics (per the structured-clone spec) whose devalue/reducer -// serialization reads prototype methods, getters, or iterators the workflow -// realm can mutate — serializing them is not provably passive, so they -// decline the fast path even when their prototype identity looks intact. -function isSlotBearingExotic(value: object): boolean { - return ( - types.isMap(value) || - types.isSet(value) || - types.isDate(value) || - types.isRegExp(value) || - types.isArrayBuffer(value) || - types.isSharedArrayBuffer(value) || - types.isDataView(value) || - types.isTypedArray(value) || - types.isBoxedPrimitive(value) || - types.isNativeError(value) || - types.isPromise(value) || - types.isArgumentsObject(value) - ); -} - function isArrayIndex(key: string): boolean { const index = Number(key); return ( @@ -184,13 +69,13 @@ function isPassiveArrayProperty( // Only own enumerable data indices are passive. Anything hidden — symbol // tags, non-enumerable properties, accessors — can be observed by // serialization dispatch (reducer probes, thenable checks, the class - // reducer) while being dropped by the clone. + // reducer) and can execute workflow code when read. return ( typeof key === 'string' && isArrayIndex(key) && descriptor.enumerable === true && 'value' in descriptor && - isPassivelyCloneable(descriptor.value, workflowGlobal, seen) + isPassive(descriptor.value, workflowGlobal, seen) ); } @@ -207,6 +92,119 @@ function isPassiveArray( ); } +// Instances of the supported built-ins must carry no own properties at all: +// serialization never reads own properties on them, but an own accessor or +// symbol could still be observed through other dispatch lookups, and clean +// instances are the overwhelmingly common case anyway. +function hasNoOwnProperties(value: object): boolean { + return Reflect.ownKeys(value).length === 0; +} + +function isPassiveMap( + value: Map, + workflowGlobal: Record, + seen: WeakSet +): boolean { + const pins = getSerializationPins(workflowGlobal); + const prototype = Object.getPrototypeOf(value); + if (prototype !== pins?.mapPrototype && prototype !== Map.prototype) { + return false; + } + if (!hasNoOwnProperties(value)) return false; + let passive = true; + // Host forEach iterates via internal slots — no realm members execute. + Map.prototype.forEach.call(value, (entryValue: unknown, key: unknown) => { + passive &&= + isPassive(key, workflowGlobal, seen) && + isPassive(entryValue, workflowGlobal, seen); + }); + return passive; +} + +function isPassiveSet( + value: Set, + workflowGlobal: Record, + seen: WeakSet +): boolean { + const pins = getSerializationPins(workflowGlobal); + const prototype = Object.getPrototypeOf(value); + if (prototype !== pins?.setPrototype && prototype !== Set.prototype) { + return false; + } + if (!hasNoOwnProperties(value)) return false; + let passive = true; + Set.prototype.forEach.call(value, (entryValue: unknown) => { + passive &&= isPassive(entryValue, workflowGlobal, seen); + }); + return passive; +} + +function isPassiveDate( + value: object, + workflowGlobal: Record +): boolean { + const pins = getSerializationPins(workflowGlobal); + const prototype = Object.getPrototypeOf(value); + return ( + (prototype === pins?.datePrototype || prototype === Date.prototype) && + hasNoOwnProperties(value) + ); +} + +function isPassiveTypedArray( + value: object, + workflowGlobal: Record +): boolean { + const pins = getSerializationPins(workflowGlobal); + const prototype = Object.getPrototypeOf(value); + if (prototype === null || types.isProxy(prototype)) return false; + const parent = Object.getPrototypeOf(prototype); + const hostTypedArrayPrototype = Object.getPrototypeOf(Uint8Array.prototype); + if ( + parent !== pins?.typedArrayPrototype && + parent !== hostTypedArrayPrototype + ) { + return false; + } + // The measured getters resolve on %TypedArray%.prototype; a shadow on the + // subclass prototype (e.g. Float32Array.prototype) would intercept them. + for (const name of ['buffer', 'byteOffset', 'byteLength']) { + if (Object.getOwnPropertyDescriptor(prototype, name)) return false; + } + // Own keys on a typed array are exactly its canonical indices. + if ( + !Reflect.ownKeys(value).every( + (key) => typeof key === 'string' && isArrayIndex(key) + ) + ) { + return false; + } + // The pinned/native getter is safe to invoke; reject SharedArrayBuffer + // backing (cross-thread mutation is unserializable either way). + const bufferGetter = ( + pins !== undefined && parent === pins.typedArrayPrototype + ? pins.typedArrayBuffer + : Object.getOwnPropertyDescriptor(hostTypedArrayPrototype, 'buffer')?.get + ) as (() => unknown) | undefined; + return ( + bufferGetter !== undefined && + !types.isSharedArrayBuffer(bufferGetter.call(value)) + ); +} + +function isPassiveArrayBuffer( + value: object, + workflowGlobal: Record +): boolean { + const pins = getSerializationPins(workflowGlobal); + const prototype = Object.getPrototypeOf(value); + return ( + (prototype === pins?.arrayBufferPrototype || + prototype === ArrayBuffer.prototype) && + hasNoOwnProperties(value) + ); +} + function isPassiveObjectProperty( object: object, key: string | symbol, @@ -218,13 +216,13 @@ function isPassiveObjectProperty( // Only own enumerable string-keyed data properties are passive. Anything // hidden — symbol tags, non-enumerable properties, accessors — can be // observed by serialization dispatch (reducer probes like `.signal`, - // thenable checks, the class reducer) while being dropped by the clone. + // thenable checks, the class reducer) and can execute workflow code. return ( typeof key === 'string' && key !== '__proto__' && descriptor.enumerable === true && 'value' in descriptor && - isPassivelyCloneable(descriptor.value, workflowGlobal, seen) + isPassive(descriptor.value, workflowGlobal, seen) ); } @@ -240,20 +238,25 @@ function isPassivePlainObject( } /** - * Whether structured cloning `value` is provably byte-equivalent to the - * ordinary VM serialization AND cannot execute workflow-owned code. + * Whether serializing `value` through the ordinary pipeline provably executes + * no workflow code and draws no workflow-realm randomness. * - * The retained VM must not observe serialization side effects that a later - * cold replay cannot reconstruct, and the durable bytes must not depend on - * `WORKFLOW_RETAINED_VM`. Only primitives, plain objects, and plain arrays - * qualify: devalue traverses them exclusively through own-property reads - * (`Object.keys`, `Object.hasOwn` + indices), so no workflow-realm prototype - * state can influence the output. Everything else — proxies, accessors, - * functions, custom classes, and built-ins like Map/Set/Date whose - * serialization consults mutable realm prototypes (iterators, getters) — - * declines the fast path and serializes through the ordinary VM traversal. + * Retained sessions keep running after suspension, so serialization side + * effects there would desync the live VM from what a cold replay + * reconstructs (replay never re-serializes an already-created step). The + * bytes themselves cannot differ by mode — serialization happens once and + * every mode reads the same `step_created` event — so passivity is the only + * property retention needs. + * + * Passive values: primitives; plain objects and arrays (own enumerable + * string-keyed data properties only — devalue traverses these purely via own + * reads); and Map/Set/Date/typed arrays/ArrayBuffer instances whose measured + * prototype members are verified pinned (see vm/serialization-pins.ts). + * Everything else — proxies, accessors, functions, custom classes, RegExp, + * hidden keys — declines, serializes exactly the same way, and the session + * falls back to ordinary replay for that boundary. */ -function isPassivelyCloneable( +function isPassive( value: unknown, workflowGlobal: Record, seen: WeakSet @@ -272,61 +275,36 @@ function isPassivelyCloneable( if (seen.has(value)) return true; seen.add(value); - if (Array.isArray(value)) { - return isPassiveArray(value, workflowGlobal, seen); + if (Array.isArray(value)) return isPassiveArray(value, workflowGlobal, seen); + if (types.isMap(value)) return isPassiveMap(value, workflowGlobal, seen); + if (types.isSet(value)) return isPassiveSet(value, workflowGlobal, seen); + if (types.isDate(value)) return isPassiveDate(value, workflowGlobal); + if (types.isTypedArray(value)) { + return isPassiveTypedArray(value, workflowGlobal); } - if (isSlotBearingExotic(value)) return false; - return isPassivePlainObject(value, workflowGlobal, seen); -} - -// Deep-copy walker-validated plain data into the pristine realm, copying -// exactly what devalue serializes: dense/sparse own indices for arrays and -// own enumerable string properties for objects. All reads are data-property -// reads — the walker already rejected everything else. -function cloneIntoPristineRealm( - value: unknown, - copies: WeakMap -): unknown { - if (value === null || typeof value !== 'object') return value; - const existing = copies.get(value); - if (existing !== undefined) return existing; - - if (Array.isArray(value)) { - const copy = new pristineRealm.Array(value.length) as unknown[]; - copies.set(value, copy); - for (const key of Reflect.ownKeys(value)) { - if (typeof key !== 'string' || !isArrayIndex(key)) continue; - copy[Number(key)] = cloneIntoPristineRealm( - (value as unknown as Record)[key], - copies - ); - } - return copy; + if (types.isArrayBuffer(value)) { + return isPassiveArrayBuffer(value, workflowGlobal); } - - const copy = new pristineRealm.Object() as Record; - copies.set(value, copy); - for (const key of Object.keys(value)) { - copy[key] = cloneIntoPristineRealm( - (value as Record)[key], - copies - ); + if ( + types.isSharedArrayBuffer(value) || + types.isRegExp(value) || + types.isDataView(value) || + types.isBoxedPrimitive(value) || + types.isNativeError(value) || + types.isPromise(value) || + types.isArgumentsObject(value) + ) { + return false; } - return copy; + return isPassivePlainObject(value, workflowGlobal, seen); } -export function prepareRetainedStepInput( +export function isRetainedSerializationPassive( value: unknown, workflowGlobal: Record -): RetainedStepInputPreparation { - if ( - !isHostDispatchPristine() || - !isPassivelyCloneable(value, workflowGlobal, new WeakSet()) - ) { - return { retainable: false }; - } - return { - retainable: true, - value: cloneIntoPristineRealm(value, new WeakMap()), - }; +): boolean { + return ( + verifySerializationPins(workflowGlobal) && + isPassive(value, workflowGlobal, new WeakSet()) + ); } diff --git a/packages/core/src/runtime/suspension-handler.ts b/packages/core/src/runtime/suspension-handler.ts index ad980e1ce7..936652c379 100644 --- a/packages/core/src/runtime/suspension-handler.ts +++ b/packages/core/src/runtime/suspension-handler.ts @@ -31,7 +31,7 @@ import * as Attribute from '../telemetry/semantic-conventions.js'; import { getAbortStreamIdFromToken } from '../util.js'; import { getMaxInlineSteps } from './constants.js'; import { type MutableEventLog, withPreconditionRetry } from './helpers.js'; -import { prepareRetainedStepInput } from './retained-step-input.js'; +import { isRetainedSerializationPassive } from './retained-step-input.js'; export interface SuspensionHandlerParams { suspension: WorkflowSuspension; @@ -497,30 +497,30 @@ export async function handleSuspension({ // racing with concurrent handlers on step execution. const createdStepCorrelationIds = new Set(); - // All-or-nothing: clones are only used when EVERY new step input in the - // batch is passive. A mixed batch would let an unsafe sibling's - // serialization (which can run getters) mutate objects a clone already - // snapshotted, changing what the other step durably receives relative to - // the ordinary in-order VM traversal. + // Serialization always runs through the one ordinary path below, so the + // durable bytes cannot depend on retention. What retention needs to know is + // whether that serialization will execute workflow code (getters, hooks, + // patched prototype members) — side effects a cold replay would not repeat. + // If any input in the batch is not provably passive, the caller demotes the + // session so the side effects land in a VM that is about to be discarded, + // exactly like the pre-retention runtime. let retainedStepInputsSafe = true; - const clonesByCorrelationId = new Map(); if (prepareForRetention) { for (const queueItem of stepItems) { if (!stepsNeedingCreation.has(queueItem.correlationId)) continue; - const prepared = prepareRetainedStepInput( - { - args: queueItem.args, - closureVars: queueItem.closureVars, - thisVal: queueItem.thisVal, - }, - suspension.globalThis - ); - if (!prepared.retainable) { + if ( + !isRetainedSerializationPassive( + { + args: queueItem.args, + closureVars: queueItem.closureVars, + thisVal: queueItem.thisVal, + }, + suspension.globalThis + ) + ) { retainedStepInputsSafe = false; - clonesByCorrelationId.clear(); break; } - clonesByCorrelationId.set(queueItem.correlationId, prepared.value); } } @@ -557,16 +557,15 @@ export async function handleSuspension({ if (stepsNeedingCreation.has(queueItem.correlationId)) { ops.push( (async () => { - const clone = clonesByCorrelationId.get(queueItem.correlationId); const dehydratedInput = await dehydrateStepArguments( - clone ?? { + { args: queueItem.args, closureVars: queueItem.closureVars, thisVal: queueItem.thisVal, }, runId, encryptionKey, - clone ? globalThis : suspension.globalThis, + suspension.globalThis, false, compression ); diff --git a/packages/core/src/vm/index.ts b/packages/core/src/vm/index.ts index ef190f64e1..5a7f5b2288 100644 --- a/packages/core/src/vm/index.ts +++ b/packages/core/src/vm/index.ts @@ -3,6 +3,7 @@ import { types } from 'node:util'; import { runInContext, createContext as vmCreateContext } from 'node:vm'; import { WorkflowRuntimeError } from '@workflow/errors'; import seedrandom from 'seedrandom'; +import { registerSerializationPins } from './serialization-pins.js'; import { installUint8ArrayBase64 } from './uint8array-base64.js'; import { createRandomUUID } from './uuid.js'; @@ -148,12 +149,11 @@ export function freezeSerializationIntrinsics(g: typeof globalThis): void { // Freeze VM-realm and session-local constructor objects — including // their prototype chains, which `Symbol.hasInstance` lookup walks (the // Date wrapper delegates statics to the unfrozen original VM Date via - // setPrototypeOf). A binding that references the shared host intrinsic - // stays unfrozen: both serialization paths dispatch through that same - // object, so a mutation cannot make them disagree (and freezing it - // would affect the whole process). Host Function.prototype's - // @@hasInstance is spec-immutable; host Object.prototype is verified - // separately (see isHostDispatchPristine). + // setPrototypeOf) — so reducer `instanceof` dispatch can never execute a + // workflow-planted hook while a retained VM's inputs serialize. A binding + // that references the shared host intrinsic stays unfrozen: freezing it + // would affect the whole process, and host-realm members run host code, + // which cannot touch retained VM state. if ( (typeof value === 'function' || (typeof value === 'object' && value !== null)) && @@ -333,6 +333,10 @@ export function createContext(options: CreateContextOptions) { g.exports = {}; (g as any).module = { exports: g.exports }; + // Capture the serialization-touched prototype members while the realm is + // provably pristine — no workflow code has run yet. + registerSerializationPins(g); + return { context, globalThis: g, diff --git a/packages/core/src/vm/serialization-pins.ts b/packages/core/src/vm/serialization-pins.ts new file mode 100644 index 0000000000..57bc4a3de2 --- /dev/null +++ b/packages/core/src/vm/serialization-pins.ts @@ -0,0 +1,115 @@ +/** + * Pinned copies of the workflow-realm prototype members that step-argument + * serialization executes, captured at context creation before any workflow + * code runs. + * + * The retained-VM fast path serializes step inputs through the ordinary + * serialization pipeline, which is only safe when that traversal provably + * executes no workflow code. For plain objects and arrays the traversal is + * own-property reads only; for the supported built-ins it also runs a small, + * measured set of prototype members (iteration protocols and getters — see + * the "touches only pinned members" test, which derives this list + * empirically). If workflow code replaced any of them, serializing that type + * would execute the replacement, so retention verifies each member is still + * the pinned original. Host-realm instances need no verification: their + * members run host code, which cannot reach the retained VM's state. + */ + +interface SerializationPins { + readonly mapPrototype: object; + readonly mapIterator: unknown; + readonly mapIteratorPrototype: object; + readonly mapIteratorNext: unknown; + readonly setPrototype: object; + readonly setIterator: unknown; + readonly setIteratorPrototype: object; + readonly setIteratorNext: unknown; + readonly datePrototype: object; + readonly dateGetDate: unknown; + readonly dateToISOString: unknown; + readonly typedArrayPrototype: object; + readonly typedArrayBuffer: unknown; + readonly typedArrayByteOffset: unknown; + readonly typedArrayByteLength: unknown; + readonly arrayBufferPrototype: object; + readonly arrayBufferByteLength: unknown; +} + +const registry = new WeakMap(); + +function ownValue(target: object, key: string | symbol): unknown { + const descriptor = Object.getOwnPropertyDescriptor(target, key); + return descriptor && 'value' in descriptor ? descriptor.value : undefined; +} + +function ownGetter(target: object, key: string | symbol): unknown { + return Object.getOwnPropertyDescriptor(target, key)?.get; +} + +export function registerSerializationPins(g: typeof globalThis): void { + const mapPrototype = g.Map.prototype; + const mapIteratorPrototype = Object.getPrototypeOf( + new g.Map()[Symbol.iterator]() + ) as object; + const setPrototype = g.Set.prototype; + const setIteratorPrototype = Object.getPrototypeOf( + new g.Set()[Symbol.iterator]() + ) as object; + // `g.Date` may already be the deterministic wrapper; its `prototype` + // data property is the realm's real Date.prototype either way. + const datePrototype = ownValue(g.Date, 'prototype') as object; + const typedArrayPrototype = Object.getPrototypeOf( + g.Uint8Array.prototype + ) as object; + const arrayBufferPrototype = g.ArrayBuffer.prototype; + + registry.set(g, { + mapPrototype, + mapIterator: ownValue(mapPrototype, Symbol.iterator), + mapIteratorPrototype, + mapIteratorNext: ownValue(mapIteratorPrototype, 'next'), + setPrototype, + setIterator: ownValue(setPrototype, Symbol.iterator), + setIteratorPrototype, + setIteratorNext: ownValue(setIteratorPrototype, 'next'), + datePrototype, + dateGetDate: ownValue(datePrototype, 'getDate'), + dateToISOString: ownValue(datePrototype, 'toISOString'), + typedArrayPrototype, + typedArrayBuffer: ownGetter(typedArrayPrototype, 'buffer'), + typedArrayByteOffset: ownGetter(typedArrayPrototype, 'byteOffset'), + typedArrayByteLength: ownGetter(typedArrayPrototype, 'byteLength'), + arrayBufferPrototype, + arrayBufferByteLength: ownGetter(arrayBufferPrototype, 'byteLength'), + }); +} + +export function getSerializationPins( + g: Record +): SerializationPins | undefined { + return registry.get(g); +} + +/** + * Whether every pinned member is still the original. All reads are + * own-descriptor reads — verification itself can never execute workflow code. + */ +export function verifySerializationPins(g: Record): boolean { + const pins = registry.get(g); + if (!pins) return false; + return ( + ownValue(pins.mapPrototype, Symbol.iterator) === pins.mapIterator && + ownValue(pins.mapIteratorPrototype, 'next') === pins.mapIteratorNext && + ownValue(pins.setPrototype, Symbol.iterator) === pins.setIterator && + ownValue(pins.setIteratorPrototype, 'next') === pins.setIteratorNext && + ownValue(pins.datePrototype, 'getDate') === pins.dateGetDate && + ownValue(pins.datePrototype, 'toISOString') === pins.dateToISOString && + ownGetter(pins.typedArrayPrototype, 'buffer') === pins.typedArrayBuffer && + ownGetter(pins.typedArrayPrototype, 'byteOffset') === + pins.typedArrayByteOffset && + ownGetter(pins.typedArrayPrototype, 'byteLength') === + pins.typedArrayByteLength && + ownGetter(pins.arrayBufferPrototype, 'byteLength') === + pins.arrayBufferByteLength + ); +} From 58514fdf176830a8cd9d72fafc7d430d15b606af Mon Sep 17 00:00:00 2001 From: Nathan Colosimo <110621881+NathanColosimo@users.noreply.github.com> Date: Tue, 21 Jul 2026 15:50:07 -0700 Subject: [PATCH 24/43] refactor(core): freeze built-in prototypes instead of pinning members (v3) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Review found the pin approach's structural hole: the class reducer READS value.constructor through Map.prototype — a data property when pristine (so member instrumentation never listed it), but executable the moment workflow code redefines it as a getter. Pinning what serialization executes misses what it reads. Freeze the accepted built-ins' prototypes wholesale (Map/Set/Date + iterator prototypes, %TypedArray% + subclass prototypes, ArrayBuffer): reads and executes are both immutable, and a patch attempt now throws loudly at the patch site instead of silently degrading. Deletes vm/serialization-pins.ts; the walker requires Object.isFrozen on the realm prototype (also covering realms where the freeze never ran). Also restores the host-dispatch pristineness check the v2 cut lost: workflow code can reach shared host constructors (exposed classes, structuredClone results) and plant workflow-realm Symbol.hasInstance hooks or WORKFLOW_SERIALIZE statics that reducers would execute during retained serialization. Host-realm built-in instances decline for the same reason; host-realm plain data (hydrated results) stays retainable. --- .changeset/retain-workflow-vm.md | 2 +- .../v5/api-reference/workflow-globals.mdx | 2 +- .../docs/v5/configuration/runtime-tuning.mdx | 2 +- .../src/runtime/retained-step-input.test.ts | 130 +++++++------- .../core/src/runtime/retained-step-input.ts | 165 +++++++++++++----- packages/core/src/vm/index.ts | 38 +++- packages/core/src/vm/serialization-pins.ts | 115 ------------ 7 files changed, 228 insertions(+), 226 deletions(-) delete mode 100644 packages/core/src/vm/serialization-pins.ts diff --git a/.changeset/retain-workflow-vm.md b/.changeset/retain-workflow-vm.md index e1853096a9..9db15ed117 100644 --- a/.changeset/retain-workflow-vm.md +++ b/.changeset/retain-workflow-vm.md @@ -3,4 +3,4 @@ 'workflow': patch --- -Retain workflow execution across passive-data inline steps within one invocation; boundaries whose argument serialization could execute workflow code fall back to replay, and `WORKFLOW_RETAINED_VM=0` disables retention. `crypto.subtle.digest` in workflow functions now computes synchronously via `node:crypto`, so digest timing is deterministic under replay. `WeakRef`, `FinalizationRegistry`, `Atomics.waitAsync`, and async `WebAssembly` compilation are no longer exposed in workflow functions (GC observation and wall-clock timing cannot be replayed), and the serialization-relevant intrinsics (`Object`/`Array`/`Function` and reducer-referenced global bindings) are frozen in the workflow sandbox. +Retain workflow execution across passive-data inline steps within one invocation; boundaries whose argument serialization could execute workflow code fall back to replay, and `WORKFLOW_RETAINED_VM=0` disables retention. `crypto.subtle.digest` in workflow functions now computes synchronously via `node:crypto`, so digest timing is deterministic under replay. `WeakRef`, `FinalizationRegistry`, `Atomics.waitAsync`, and async `WebAssembly` compilation are no longer exposed in workflow functions (GC observation and wall-clock timing cannot be replayed), and the serialization-relevant intrinsics (`Object`/`Array`/`Function`, the prototypes of `Map`/`Set`/`Date`/typed arrays/`ArrayBuffer`, and reducer-referenced global bindings) are frozen in the workflow sandbox. diff --git a/docs/content/docs/v5/api-reference/workflow-globals.mdx b/docs/content/docs/v5/api-reference/workflow-globals.mdx index 59e8458b22..6b49709597 100644 --- a/docs/content/docs/v5/api-reference/workflow-globals.mdx +++ b/docs/content/docs/v5/api-reference/workflow-globals.mdx @@ -103,4 +103,4 @@ The following are **not available** in workflow functions. Move this logic to [s - **`WeakRef` and `FinalizationRegistry`**: garbage-collection timing is not deterministic, so observing it would make workflow code impossible to replay. (`WeakMap` and `WeakSet` remain available — they do not expose GC state.) - **`Atomics.waitAsync`**: a wall-clock timer, which cannot be replayed. Use [`sleep()`](/docs/api-reference/workflow/sleep) instead. - **Async `WebAssembly` compilation** (`compile`, `instantiate`, `compileStreaming`, `instantiateStreaming`): resolves on compile-thread timing. The synchronous `new WebAssembly.Module()` and `new WebAssembly.Instance()` constructors remain available. -- **Mutating core intrinsics**: `Object`, `Array`, and `Function` (constructors and prototypes), and the global bindings for serialization-relevant built-ins (`Map`, `Set`, `Date`, `RegExp`, typed arrays, `Headers`, `URL`, and similar) are frozen. Serialization dispatches on these, so redefining them would make replay unfaithful. +- **Mutating core intrinsics**: `Object`, `Array`, and `Function` (constructors and prototypes), the prototypes of `Map`, `Set`, `Date`, typed arrays, and `ArrayBuffer`, and the global bindings for serialization-relevant built-ins (`RegExp`, `Headers`, `URL`, and similar) are frozen. Serialization reads through these, so redefining them would make replay unfaithful. diff --git a/docs/content/docs/v5/configuration/runtime-tuning.mdx b/docs/content/docs/v5/configuration/runtime-tuning.mdx index 17322d7ca9..b1e881f0f2 100644 --- a/docs/content/docs/v5/configuration/runtime-tuning.mdx +++ b/docs/content/docs/v5/configuration/runtime-tuning.mdx @@ -80,7 +80,7 @@ For example, a workflow can run a 10-minute inline step even with `WORKFLOW_REPL - Default: enabled - Keeps the suspended workflow VM alive across inline steps within one invocation, so each iteration of the inline loop appends only the newly written events instead of replaying the whole event log in a fresh VM. - Suspensions involving hooks, waits, or attributes — and any replay divergence — always fall back to a full replay. -- Step inputs made of plain data (objects, arrays, primitives) and standard built-ins (`Map`, `Set`, `Date`, typed arrays, `ArrayBuffer`) remain retainable. Anything whose serialization could execute workflow code — accessors, proxies, functions, custom class serializers, `RegExp` — falls back to replay for that boundary, as does any input if the prototype members serialization relies on have been modified. +- Step inputs made of plain data (objects, arrays, primitives) and standard built-ins (`Map`, `Set`, `Date`, typed arrays, `ArrayBuffer`) remain retainable. Anything whose serialization could execute workflow code — accessors, proxies, functions, custom class serializers, `RegExp` — falls back to replay for that boundary. - Set `0` or `false` to replay from scratch in a fresh VM on every iteration. ### `WORKFLOW_INLINE_OWNERSHIP` diff --git a/packages/core/src/runtime/retained-step-input.test.ts b/packages/core/src/runtime/retained-step-input.test.ts index 43c0ab46df..b84492deff 100644 --- a/packages/core/src/runtime/retained-step-input.test.ts +++ b/packages/core/src/runtime/retained-step-input.test.ts @@ -1,13 +1,13 @@ import vm from 'node:vm'; import { describe, expect, it } from 'vitest'; import { dehydrateStepArguments } from '../serialization.js'; -import { createContext } from '../vm/index.js'; +import { createContext, freezeSerializationIntrinsics } from '../vm/index.js'; import { isRetainedSerializationPassive } from './retained-step-input.js'; const seed = 'retained-step-input'; const fixedTimestamp = 1_700_000_000_000; -function makeContext() { +function makeContext({ freeze = true } = {}) { const { context, globalThis: workflowGlobal } = createContext({ seed, fixedTimestamp, @@ -27,6 +27,7 @@ function makeContext() { (workflowGlobal as any)[name] = (globalThis as any)[name]; } } + if (freeze) freezeSerializationIntrinsics(workflowGlobal); return { context, workflowGlobal }; } @@ -45,7 +46,7 @@ describe('isRetainedSerializationPassive', () => { expect(isRetainedSerializationPassive(value, workflowGlobal)).toBe(true); }); - it('accepts the supported built-ins with pristine pins', () => { + it('accepts the supported built-ins on a frozen realm', () => { const { context, workflowGlobal } = makeContext(); for (const expression of [ 'new Map([["k", { ok: true }]])', @@ -61,18 +62,23 @@ describe('isRetainedSerializationPassive', () => { } }); - it('accepts host-realm instances (hydrated step results)', () => { + it('accepts host-realm plain data but not host-realm built-ins', () => { const { workflowGlobal } = makeContext(); - const value = { - map: new Map([['k', 1]]), - date: new Date(1234), - bytes: new Uint8Array([9]), - }; - - expect(isRetainedSerializationPassive(value, workflowGlobal)).toBe(true); + // Hydrated step results are host-realm plain objects/arrays. + expect( + isRetainedSerializationPassive({ nested: [{ ok: true }] }, workflowGlobal) + ).toBe(true); + // Host built-in prototypes cannot be frozen (process-shared) and are + // reachable from workflow code, so host-realm instances decline. + expect( + isRetainedSerializationPassive(new Map([['k', 1]]), workflowGlobal) + ).toBe(false); + expect(isRetainedSerializationPassive(new Date(0), workflowGlobal)).toBe( + false + ); }); - it('declines types whose serialization is not pinned', () => { + it('declines types whose serialization surface is not frozen', () => { const { context, workflowGlobal } = makeContext(); for (const expression of [ '/workflow/gi', @@ -179,66 +185,64 @@ describe('isRetainedSerializationPassive', () => { } }); - it('declines everything once a pinned member is patched', () => { + it('freezing makes prototype patches impossible, so retention persists', () => { const { context, workflowGlobal } = makeContext(); - const plain = vm.runInContext('({ ok: true })', context); - expect(isRetainedSerializationPassive(plain, workflowGlobal)).toBe(true); + const map = vm.runInContext('new Map([["k", 1]])', context); + expect(isRetainedSerializationPassive(map, workflowGlobal)).toBe(true); - vm.runInContext( - `globalThis.__origIterator = Map.prototype[Symbol.iterator]; - Map.prototype[Symbol.iterator] = function () { return globalThis.__origIterator.call(this); };`, - context - ); - expect(isRetainedSerializationPassive(plain, workflowGlobal)).toBe(false); + // Redefining a member the serializer reads throws on the frozen prototype. + for (const attempt of [ + '"use strict"; Object.defineProperty(Map.prototype, "constructor", { get() { return 1; } })', + '"use strict"; Map.prototype[Symbol.iterator] = function () {};', + '"use strict"; Date.prototype.toISOString = function () { return "x"; };', + '"use strict"; Object.defineProperty(Float32Array.prototype, "buffer", { get() {} })', + ]) { + expect(() => vm.runInContext(attempt, context)).toThrow( + /not extensible|read only|Cannot redefine/ + ); + } + // Nothing changed, so the built-ins remain retainable. + expect(isRetainedSerializationPassive(map, workflowGlobal)).toBe(true); + }); - vm.runInContext( - 'Map.prototype[Symbol.iterator] = globalThis.__origIterator;', - context - ); + it('declines built-ins when the realm was never frozen', () => { + const { context, workflowGlobal } = makeContext({ freeze: false }); + const map = vm.runInContext('new Map()', context); + expect(isRetainedSerializationPassive(map, workflowGlobal)).toBe(false); + // Plain data does not depend on the built-in prototypes. + const plain = vm.runInContext('({ ok: true })', context); expect(isRetainedSerializationPassive(plain, workflowGlobal)).toBe(true); }); - it('declines per pinned member: iterator next, Date methods', () => { - for (const patch of [ - 'Object.getPrototypeOf((new Map())[Symbol.iterator]()).next = function () {};', - 'Object.getPrototypeOf((new Set())[Symbol.iterator]()).next = function () {};', - 'Date.prototype.toISOString = function () { return "spoofed"; };', - 'Date.prototype.getDate = function () { return 1; };', - ]) { - const { context, workflowGlobal } = makeContext(); - const plain = vm.runInContext('({ ok: true })', context); - vm.runInContext(patch, context); + it('declines retention while a host dispatch constructor is hooked', () => { + const { context, workflowGlobal } = makeContext(); + const plain = vm.runInContext('({ ok: true })', context); + expect(isRetainedSerializationPassive(plain, workflowGlobal)).toBe(true); + + Object.defineProperty(Headers, Symbol.hasInstance, { + value: () => false, + configurable: true, + }); + try { expect(isRetainedSerializationPassive(plain, workflowGlobal)).toBe(false); + } finally { + delete (Headers as any)[Symbol.hasInstance]; } - }); - - it('declines typed arrays whose subclass prototype shadows a pinned getter', () => { - const { context, workflowGlobal } = makeContext(); - const value = vm.runInContext( - `(() => { - Object.defineProperty(Float32Array.prototype, "buffer", { - get() { return new ArrayBuffer(0); }, configurable: true, - }); - return new Float32Array([1.5]); - })()`, - context - ); - expect(isRetainedSerializationPassive(value, workflowGlobal)).toBe(false); - // Pins themselves are intact, so unrelated values stay retainable. - const plain = vm.runInContext('({ ok: true })', context); expect(isRetainedSerializationPassive(plain, workflowGlobal)).toBe(true); }); }); -describe('serialization touches only pinned members', () => { - // THE coupling test: the pin list in vm/serialization-pins.ts is only sound - // if it covers every prototype member `dehydrateStepArguments` executes for - // the supported built-ins. Wrap every configurable member on the relevant - // prototypes with a recorder and assert the serializer hits nothing beyond - // the pinned set. If serde changes what it touches, this fails loudly — - // update the pin list (and this list) together. - const PINNED = new Set([ +describe('serialization touches only the frozen surface', () => { + // THE coupling test: retention is only sound if every prototype member + // `dehydrateStepArguments` executes for the supported built-ins lives on an + // object `freezeSerializationIntrinsics` freezes. Wrap every configurable + // member on the relevant prototypes (in an unfrozen realm) with a recorder + // and assert the serializer hits nothing beyond this measured set — every + // entry of which is on a frozen prototype in production. If serde starts + // touching something new, this fails loudly: extend the freeze (and this + // list) together. + const FROZEN_SURFACE = new Set([ 'Map.prototype.Symbol(Symbol.iterator)', '%MapIteratorPrototype%.next', 'Set.prototype.Symbol(Symbol.iterator)', @@ -252,7 +256,8 @@ describe('serialization touches only pinned members', () => { ]); it('for Map, Set, Date, typed arrays, and ArrayBuffer', async () => { - const { context, workflowGlobal } = makeContext(); + // Unfrozen realm: the recorders themselves need to redefine members. + const { context, workflowGlobal } = makeContext({ freeze: false }); const g = workflowGlobal as any; const touched = new Set(); @@ -329,7 +334,10 @@ describe('serialization touches only pinned members', () => { false ); for (const name of touched) { - expect(PINNED, `unpinned member executed: ${name}`).toContain(name); + expect( + FROZEN_SURFACE, + `member outside the frozen surface executed: ${name}` + ).toContain(name); } } }); diff --git a/packages/core/src/runtime/retained-step-input.ts b/packages/core/src/runtime/retained-step-input.ts index 0e478cd8a2..9f49fe2954 100644 --- a/packages/core/src/runtime/retained-step-input.ts +++ b/packages/core/src/runtime/retained-step-input.ts @@ -1,8 +1,87 @@ import { types } from 'node:util'; -import { - getSerializationPins, - verifySerializationPins, -} from '../vm/serialization-pins.js'; +import { WORKFLOW_SERIALIZE } from '@workflow/serde'; + +// Host constructors that serialization dispatches on (`value instanceof +// global.X`), plus Object/Array, whose statics and prototypes the class +// reducer and devalue's tag lookup read for host-prototype values (hydrated +// step results are host-realm plain objects). The workflow VM's own +// intrinsics are frozen (see vm/index.ts), but host intrinsics are shared +// with the whole process and cannot be frozen — and workflow code can reach +// them (exposed host classes, `structuredClone` results) and plant +// workflow-realm hooks. Verified instead: any dirt declines retention, so a +// planted hook can never execute while a retained VM's inputs serialize. +const HOST_DISPATCH_CONSTRUCTORS = [ + 'Object', + 'Array', + 'Function', + 'Map', + 'Set', + 'Date', + 'RegExp', + 'ArrayBuffer', + 'SharedArrayBuffer', + 'DataView', + 'Int8Array', + 'Uint8Array', + 'Uint8ClampedArray', + 'Int16Array', + 'Uint16Array', + 'Int32Array', + 'Uint32Array', + 'Float16Array', + 'Float32Array', + 'Float64Array', + 'BigInt64Array', + 'BigUint64Array', + 'Headers', + 'URL', + 'URLSearchParams', + 'DOMException', + 'AbortController', + 'AbortSignal', + 'Request', + 'Response', + 'ReadableStream', + 'WritableStream', + 'TransformStream', +] as const; + +function hasOwn(target: object, key: string | symbol): boolean { + return Object.getOwnPropertyDescriptor(target, key) !== undefined; +} + +function isHostDispatchPristine(): boolean { + const hostGlobal = globalThis as unknown as Record; + for (const name of HOST_DISPATCH_CONSTRUCTORS) { + const constructor = hostGlobal[name]; + if (constructor === undefined) continue; + if (hasOwn(constructor as object, Symbol.hasInstance)) return false; + } + for (const constructor of [Object, Array]) { + if ( + hasOwn(constructor, WORKFLOW_SERIALIZE) || + hasOwn(constructor, 'classId') + ) { + return false; + } + } + for (const [prototype, constructor] of [ + [Object.prototype, Object], + [Array.prototype, Array], + ] as const) { + if ( + hasOwn(prototype, Symbol.toStringTag) || + ownDataProperty(prototype, 'constructor') !== constructor + ) { + return false; + } + } + // Constructor prototype chains end at host Object.prototype, where an + // added @@hasInstance would be found by dispatch lookup. (Host + // Function.prototype's @@hasInstance is spec non-configurable.) + if (hasOwn(Object.prototype, Symbol.hasInstance)) return false; + return true; +} // Own data-property read that never performs a property Get — workflow code // can redefine its globals (or their `prototype` slots) with accessors, and @@ -32,6 +111,23 @@ function constructorPrototype( : undefined; } +// The workflow realm's prototype for a supported built-in, but only once the +// sandbox froze it: serialization both executes members on these prototypes +// (iterators, getters) and reads others (`constructor`), so a mutable +// prototype — including one in a realm where freezeSerializationIntrinsics +// never ran — is not provably passive. Host-realm instances of these +// built-ins decline for the same reason: host prototypes cannot be frozen +// and are reachable from workflow code (e.g. via structuredClone results). +function frozenRealmPrototype( + workflowGlobal: Record, + constructorName: string +): object | undefined { + const prototype = constructorPrototype(workflowGlobal, constructorName); + return prototype !== undefined && Object.isFrozen(prototype) + ? prototype + : undefined; +} + function hasAllowedPrototype( value: object, workflowGlobal: Record, @@ -105,11 +201,8 @@ function isPassiveMap( workflowGlobal: Record, seen: WeakSet ): boolean { - const pins = getSerializationPins(workflowGlobal); const prototype = Object.getPrototypeOf(value); - if (prototype !== pins?.mapPrototype && prototype !== Map.prototype) { - return false; - } + if (prototype !== frozenRealmPrototype(workflowGlobal, 'Map')) return false; if (!hasNoOwnProperties(value)) return false; let passive = true; // Host forEach iterates via internal slots — no realm members execute. @@ -126,11 +219,8 @@ function isPassiveSet( workflowGlobal: Record, seen: WeakSet ): boolean { - const pins = getSerializationPins(workflowGlobal); const prototype = Object.getPrototypeOf(value); - if (prototype !== pins?.setPrototype && prototype !== Set.prototype) { - return false; - } + if (prototype !== frozenRealmPrototype(workflowGlobal, 'Set')) return false; if (!hasNoOwnProperties(value)) return false; let passive = true; Set.prototype.forEach.call(value, (entryValue: unknown) => { @@ -143,10 +233,9 @@ function isPassiveDate( value: object, workflowGlobal: Record ): boolean { - const pins = getSerializationPins(workflowGlobal); const prototype = Object.getPrototypeOf(value); return ( - (prototype === pins?.datePrototype || prototype === Date.prototype) && + prototype === frozenRealmPrototype(workflowGlobal, 'Date') && hasNoOwnProperties(value) ); } @@ -155,21 +244,24 @@ function isPassiveTypedArray( value: object, workflowGlobal: Record ): boolean { - const pins = getSerializationPins(workflowGlobal); const prototype = Object.getPrototypeOf(value); - if (prototype === null || types.isProxy(prototype)) return false; - const parent = Object.getPrototypeOf(prototype); - const hostTypedArrayPrototype = Object.getPrototypeOf(Uint8Array.prototype); if ( - parent !== pins?.typedArrayPrototype && - parent !== hostTypedArrayPrototype + prototype === null || + types.isProxy(prototype) || + !Object.isFrozen(prototype) ) { return false; } - // The measured getters resolve on %TypedArray%.prototype; a shadow on the - // subclass prototype (e.g. Float32Array.prototype) would intercept them. - for (const name of ['buffer', 'byteOffset', 'byteLength']) { - if (Object.getOwnPropertyDescriptor(prototype, name)) return false; + // The subclass prototype must chain to the realm's frozen %TypedArray% + // parent, where the buffer/byteOffset/byteLength getters live. + const uint8Prototype = frozenRealmPrototype(workflowGlobal, 'Uint8Array'); + if (uint8Prototype === undefined) return false; + const typedArrayPrototype = Object.getPrototypeOf(uint8Prototype); + if ( + Object.getPrototypeOf(prototype) !== typedArrayPrototype || + !Object.isFrozen(typedArrayPrototype) + ) { + return false; } // Own keys on a typed array are exactly its canonical indices. if ( @@ -179,28 +271,18 @@ function isPassiveTypedArray( ) { return false; } - // The pinned/native getter is safe to invoke; reject SharedArrayBuffer - // backing (cross-thread mutation is unserializable either way). - const bufferGetter = ( - pins !== undefined && parent === pins.typedArrayPrototype - ? pins.typedArrayBuffer - : Object.getOwnPropertyDescriptor(hostTypedArrayPrototype, 'buffer')?.get - ) as (() => unknown) | undefined; - return ( - bufferGetter !== undefined && - !types.isSharedArrayBuffer(bufferGetter.call(value)) - ); + // Frozen chain ⇒ the getters are the originals; safe to invoke. Reject + // SharedArrayBuffer backing (cross-thread mutation is unserializable). + return !types.isSharedArrayBuffer((value as Uint8Array).buffer); } function isPassiveArrayBuffer( value: object, workflowGlobal: Record ): boolean { - const pins = getSerializationPins(workflowGlobal); const prototype = Object.getPrototypeOf(value); return ( - (prototype === pins?.arrayBufferPrototype || - prototype === ArrayBuffer.prototype) && + prototype === frozenRealmPrototype(workflowGlobal, 'ArrayBuffer') && hasNoOwnProperties(value) ); } @@ -250,8 +332,8 @@ function isPassivePlainObject( * * Passive values: primitives; plain objects and arrays (own enumerable * string-keyed data properties only — devalue traverses these purely via own - * reads); and Map/Set/Date/typed arrays/ArrayBuffer instances whose measured - * prototype members are verified pinned (see vm/serialization-pins.ts). + * reads); and workflow-realm Map/Set/Date/typed arrays/ArrayBuffer whose + * prototypes the sandbox froze (see freezeSerializationIntrinsics). * Everything else — proxies, accessors, functions, custom classes, RegExp, * hidden keys — declines, serializes exactly the same way, and the session * falls back to ordinary replay for that boundary. @@ -304,7 +386,6 @@ export function isRetainedSerializationPassive( workflowGlobal: Record ): boolean { return ( - verifySerializationPins(workflowGlobal) && - isPassive(value, workflowGlobal, new WeakSet()) + isHostDispatchPristine() && isPassive(value, workflowGlobal, new WeakSet()) ); } diff --git a/packages/core/src/vm/index.ts b/packages/core/src/vm/index.ts index 5a7f5b2288..376fd01961 100644 --- a/packages/core/src/vm/index.ts +++ b/packages/core/src/vm/index.ts @@ -3,7 +3,6 @@ import { types } from 'node:util'; import { runInContext, createContext as vmCreateContext } from 'node:vm'; import { WorkflowRuntimeError } from '@workflow/errors'; import seedrandom from 'seedrandom'; -import { registerSerializationPins } from './serialization-pins.js'; import { installUint8ArrayBase64 } from './uint8array-base64.js'; import { createRandomUUID } from './uuid.js'; @@ -141,6 +140,39 @@ export function freezeSerializationIntrinsics(g: typeof globalThis): void { // The typed-array constructors dispatch `instanceof` through their shared // %TypedArray% parent. Object.freeze(Object.getPrototypeOf(g.Uint8Array)); + // Prototypes of the built-ins the retained fast path accepts as step + // arguments (see runtime/retained-step-input.ts): serialization both + // executes members on them (iteration protocols, getters) and merely READS + // others (`constructor` via the class reducer), so pinning individual + // members is not enough — a replaced data property becomes executable the + // moment it is redefined as an accessor. Frozen wholesale, nothing on them + // can ever be workflow-planted. RegExp/DataView prototypes stay mutable; + // those types always take the ordinary replay path. + Object.freeze(g.Map.prototype); + Object.freeze(Object.getPrototypeOf(new g.Map()[Symbol.iterator]())); + Object.freeze(g.Set.prototype); + Object.freeze(Object.getPrototypeOf(new g.Set()[Symbol.iterator]())); + // biome-ignore lint/style/noNonNullAssertion: Date always has a prototype + Object.freeze(Object.getOwnPropertyDescriptor(g.Date, 'prototype')!.value); + Object.freeze(Object.getPrototypeOf(g.Uint8Array.prototype)); + for (const name of [ + 'Int8Array', + 'Uint8Array', + 'Uint8ClampedArray', + 'Int16Array', + 'Uint16Array', + 'Int32Array', + 'Uint32Array', + 'Float16Array', + 'Float32Array', + 'Float64Array', + 'BigInt64Array', + 'BigUint64Array', + ]) { + const constructor = (g as unknown as Record)[name]; + if (constructor !== undefined) Object.freeze(constructor.prototype); + } + Object.freeze(g.ArrayBuffer.prototype); const hostGlobal = globalThis as unknown as Record; for (const name of SERIALIZATION_BINDINGS) { const descriptor = Object.getOwnPropertyDescriptor(g, name); @@ -333,10 +365,6 @@ export function createContext(options: CreateContextOptions) { g.exports = {}; (g as any).module = { exports: g.exports }; - // Capture the serialization-touched prototype members while the realm is - // provably pristine — no workflow code has run yet. - registerSerializationPins(g); - return { context, globalThis: g, diff --git a/packages/core/src/vm/serialization-pins.ts b/packages/core/src/vm/serialization-pins.ts deleted file mode 100644 index 57bc4a3de2..0000000000 --- a/packages/core/src/vm/serialization-pins.ts +++ /dev/null @@ -1,115 +0,0 @@ -/** - * Pinned copies of the workflow-realm prototype members that step-argument - * serialization executes, captured at context creation before any workflow - * code runs. - * - * The retained-VM fast path serializes step inputs through the ordinary - * serialization pipeline, which is only safe when that traversal provably - * executes no workflow code. For plain objects and arrays the traversal is - * own-property reads only; for the supported built-ins it also runs a small, - * measured set of prototype members (iteration protocols and getters — see - * the "touches only pinned members" test, which derives this list - * empirically). If workflow code replaced any of them, serializing that type - * would execute the replacement, so retention verifies each member is still - * the pinned original. Host-realm instances need no verification: their - * members run host code, which cannot reach the retained VM's state. - */ - -interface SerializationPins { - readonly mapPrototype: object; - readonly mapIterator: unknown; - readonly mapIteratorPrototype: object; - readonly mapIteratorNext: unknown; - readonly setPrototype: object; - readonly setIterator: unknown; - readonly setIteratorPrototype: object; - readonly setIteratorNext: unknown; - readonly datePrototype: object; - readonly dateGetDate: unknown; - readonly dateToISOString: unknown; - readonly typedArrayPrototype: object; - readonly typedArrayBuffer: unknown; - readonly typedArrayByteOffset: unknown; - readonly typedArrayByteLength: unknown; - readonly arrayBufferPrototype: object; - readonly arrayBufferByteLength: unknown; -} - -const registry = new WeakMap(); - -function ownValue(target: object, key: string | symbol): unknown { - const descriptor = Object.getOwnPropertyDescriptor(target, key); - return descriptor && 'value' in descriptor ? descriptor.value : undefined; -} - -function ownGetter(target: object, key: string | symbol): unknown { - return Object.getOwnPropertyDescriptor(target, key)?.get; -} - -export function registerSerializationPins(g: typeof globalThis): void { - const mapPrototype = g.Map.prototype; - const mapIteratorPrototype = Object.getPrototypeOf( - new g.Map()[Symbol.iterator]() - ) as object; - const setPrototype = g.Set.prototype; - const setIteratorPrototype = Object.getPrototypeOf( - new g.Set()[Symbol.iterator]() - ) as object; - // `g.Date` may already be the deterministic wrapper; its `prototype` - // data property is the realm's real Date.prototype either way. - const datePrototype = ownValue(g.Date, 'prototype') as object; - const typedArrayPrototype = Object.getPrototypeOf( - g.Uint8Array.prototype - ) as object; - const arrayBufferPrototype = g.ArrayBuffer.prototype; - - registry.set(g, { - mapPrototype, - mapIterator: ownValue(mapPrototype, Symbol.iterator), - mapIteratorPrototype, - mapIteratorNext: ownValue(mapIteratorPrototype, 'next'), - setPrototype, - setIterator: ownValue(setPrototype, Symbol.iterator), - setIteratorPrototype, - setIteratorNext: ownValue(setIteratorPrototype, 'next'), - datePrototype, - dateGetDate: ownValue(datePrototype, 'getDate'), - dateToISOString: ownValue(datePrototype, 'toISOString'), - typedArrayPrototype, - typedArrayBuffer: ownGetter(typedArrayPrototype, 'buffer'), - typedArrayByteOffset: ownGetter(typedArrayPrototype, 'byteOffset'), - typedArrayByteLength: ownGetter(typedArrayPrototype, 'byteLength'), - arrayBufferPrototype, - arrayBufferByteLength: ownGetter(arrayBufferPrototype, 'byteLength'), - }); -} - -export function getSerializationPins( - g: Record -): SerializationPins | undefined { - return registry.get(g); -} - -/** - * Whether every pinned member is still the original. All reads are - * own-descriptor reads — verification itself can never execute workflow code. - */ -export function verifySerializationPins(g: Record): boolean { - const pins = registry.get(g); - if (!pins) return false; - return ( - ownValue(pins.mapPrototype, Symbol.iterator) === pins.mapIterator && - ownValue(pins.mapIteratorPrototype, 'next') === pins.mapIteratorNext && - ownValue(pins.setPrototype, Symbol.iterator) === pins.setIterator && - ownValue(pins.setIteratorPrototype, 'next') === pins.setIteratorNext && - ownValue(pins.datePrototype, 'getDate') === pins.dateGetDate && - ownValue(pins.datePrototype, 'toISOString') === pins.dateToISOString && - ownGetter(pins.typedArrayPrototype, 'buffer') === pins.typedArrayBuffer && - ownGetter(pins.typedArrayPrototype, 'byteOffset') === - pins.typedArrayByteOffset && - ownGetter(pins.typedArrayPrototype, 'byteLength') === - pins.typedArrayByteLength && - ownGetter(pins.arrayBufferPrototype, 'byteLength') === - pins.arrayBufferByteLength - ); -} From 93aa3a5b88dce017341de90a7177550261e01b56 Mon Sep 17 00:00:00 2001 From: Nathan Colosimo <110621881+NathanColosimo@users.noreply.github.com> Date: Tue, 21 Jul 2026 16:05:01 -0700 Subject: [PATCH 25/43] fix(core): harden the passivity checker's own execution surface MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - Capture Map/Set forEach and the %TypedArray% buffer getter as module- load primordials: the checker previously invoked live host methods that workflow code can reach (structuredClone(new Map()).constructor) and replace with delegating workflow-realm closures. - Typed arrays must have one of the realm's real frozen subclass prototypes by identity — 'frozen and chains to %TypedArray%' admitted manufactured frozen hostile prototypes with delegating buffer getters. --- .../src/runtime/retained-step-input.test.ts | 49 +++++++++++++++ .../core/src/runtime/retained-step-input.ts | 61 +++++++++++++------ 2 files changed, 90 insertions(+), 20 deletions(-) diff --git a/packages/core/src/runtime/retained-step-input.test.ts b/packages/core/src/runtime/retained-step-input.test.ts index b84492deff..58573edc22 100644 --- a/packages/core/src/runtime/retained-step-input.test.ts +++ b/packages/core/src/runtime/retained-step-input.test.ts @@ -342,3 +342,52 @@ describe('serialization touches only the frozen surface', () => { } }); }); + +describe('checker execution surface', () => { + it('never invokes live host collection methods', () => { + const { context, workflowGlobal } = makeContext(); + const map = vm.runInContext('new Map([["k", 1]])', context); + + let invoked = 0; + const original = Map.prototype.forEach; + // Simulate workflow code having replaced the reachable host method. + Map.prototype.forEach = function ( + this: Map, + ...args: [any] + ) { + invoked++; + return original.apply(this, args); + }; + try { + // The checker uses the module-captured primordial: same verdict, + // replaced method never runs. + expect(isRetainedSerializationPassive(map, workflowGlobal)).toBe(true); + expect(invoked).toBe(0); + } finally { + Map.prototype.forEach = original; + } + }); + + it('declines typed arrays re-prototyped onto a frozen hostile prototype', () => { + const { context, workflowGlobal } = makeContext(); + const value = vm.runInContext( + `(() => { + globalThis.__retainedTestCalls = 0; + const realGetter = Object.getOwnPropertyDescriptor( + Object.getPrototypeOf(Uint8Array.prototype), "buffer").get; + const hostile = Object.create( + Object.getPrototypeOf(Uint8Array.prototype), + { buffer: { get() { globalThis.__retainedTestCalls++; return realGetter.call(this); } } } + ); + Object.freeze(hostile); + const ta = new Uint8Array([1, 2]); + Object.setPrototypeOf(ta, hostile); + return ta; + })()`, + context + ); + + expect(isRetainedSerializationPassive(value, workflowGlobal)).toBe(false); + expect(vm.runInContext('globalThis.__retainedTestCalls', context)).toBe(0); + }); +}); diff --git a/packages/core/src/runtime/retained-step-input.ts b/packages/core/src/runtime/retained-step-input.ts index 9f49fe2954..0da2ed82e3 100644 --- a/packages/core/src/runtime/retained-step-input.ts +++ b/packages/core/src/runtime/retained-step-input.ts @@ -46,6 +46,33 @@ const HOST_DISPATCH_CONSTRUCTORS = [ 'TransformStream', ] as const; +// Host primordials captured at module load — before any workflow code can +// exist in the process — so the checker itself never invokes a live host +// method workflow code could have replaced (host constructors are reachable +// via e.g. `structuredClone(new Map()).constructor`). +const hostMapForEach = Map.prototype.forEach; +const hostSetForEach = Set.prototype.forEach; +// biome-ignore lint/style/noNonNullAssertion: the %TypedArray% buffer getter always exists +const hostTypedArrayBuffer = Object.getOwnPropertyDescriptor( + Object.getPrototypeOf(Uint8Array.prototype), + 'buffer' +)!.get!; + +const TYPED_ARRAY_CONSTRUCTORS = [ + 'Int8Array', + 'Uint8Array', + 'Uint8ClampedArray', + 'Int16Array', + 'Uint16Array', + 'Int32Array', + 'Uint32Array', + 'Float16Array', + 'Float32Array', + 'Float64Array', + 'BigInt64Array', + 'BigUint64Array', +] as const; + function hasOwn(target: object, key: string | symbol): boolean { return Object.getOwnPropertyDescriptor(target, key) !== undefined; } @@ -205,8 +232,9 @@ function isPassiveMap( if (prototype !== frozenRealmPrototype(workflowGlobal, 'Map')) return false; if (!hasNoOwnProperties(value)) return false; let passive = true; - // Host forEach iterates via internal slots — no realm members execute. - Map.prototype.forEach.call(value, (entryValue: unknown, key: unknown) => { + // The captured host forEach iterates via internal slots — no realm + // members (and no live, replaceable host members) execute. + hostMapForEach.call(value, (entryValue: unknown, key: unknown) => { passive &&= isPassive(key, workflowGlobal, seen) && isPassive(entryValue, workflowGlobal, seen); @@ -223,7 +251,7 @@ function isPassiveSet( if (prototype !== frozenRealmPrototype(workflowGlobal, 'Set')) return false; if (!hasNoOwnProperties(value)) return false; let passive = true; - Set.prototype.forEach.call(value, (entryValue: unknown) => { + hostSetForEach.call(value, (entryValue: unknown) => { passive &&= isPassive(entryValue, workflowGlobal, seen); }); return passive; @@ -244,22 +272,15 @@ function isPassiveTypedArray( value: object, workflowGlobal: Record ): boolean { + // Identity against the finite set of the realm's real (frozen) typed-array + // prototypes — NOT "anything chaining to %TypedArray%": a workflow can + // manufacture a frozen hostile prototype with a delegating `buffer` getter + // and setPrototypeOf a real typed array onto it. const prototype = Object.getPrototypeOf(value); if ( - prototype === null || - types.isProxy(prototype) || - !Object.isFrozen(prototype) - ) { - return false; - } - // The subclass prototype must chain to the realm's frozen %TypedArray% - // parent, where the buffer/byteOffset/byteLength getters live. - const uint8Prototype = frozenRealmPrototype(workflowGlobal, 'Uint8Array'); - if (uint8Prototype === undefined) return false; - const typedArrayPrototype = Object.getPrototypeOf(uint8Prototype); - if ( - Object.getPrototypeOf(prototype) !== typedArrayPrototype || - !Object.isFrozen(typedArrayPrototype) + !TYPED_ARRAY_CONSTRUCTORS.some( + (name) => prototype === frozenRealmPrototype(workflowGlobal, name) + ) ) { return false; } @@ -271,9 +292,9 @@ function isPassiveTypedArray( ) { return false; } - // Frozen chain ⇒ the getters are the originals; safe to invoke. Reject - // SharedArrayBuffer backing (cross-thread mutation is unserializable). - return !types.isSharedArrayBuffer((value as Uint8Array).buffer); + // Read the backing buffer via the captured host getter (internal slots + // work cross-realm); reject SharedArrayBuffer backing. + return !types.isSharedArrayBuffer(hostTypedArrayBuffer.call(value)); } function isPassiveArrayBuffer( From fb78452b5e21520c9493dd88c103c3dc60d40a5a Mon Sep 17 00:00:00 2001 From: Nathan Colosimo <110621881+NathanColosimo@users.noreply.github.com> Date: Tue, 21 Jul 2026 16:22:58 -0700 Subject: [PATCH 26/43] fix(core): checker uses module-load primordials; verify inherited serializer statics MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - The walker resolved Object.getOwnPropertyDescriptor, Reflect.ownKeys, Array.isArray, Number/String helpers, and Object.getPrototypeOf/isFrozen from live host globals workflow code can reach and replace; all are now module-load captures, so the checker can never execute a planted delegate. - The class reducer reads cls[WORKFLOW_SERIALIZE]/cls.classId as inherited Gets, so isHostDispatchPristine now also verifies host Function.prototype and Object.prototype carry no serializer statics. Generic replacement of shared host statics (Object.keys, Array.from, …) via realm escape remains the documented host-reachability boundary, tracked by the realm-local intrinsics follow-up. --- .../src/runtime/retained-step-input.test.ts | 47 +++++++++++++++ .../core/src/runtime/retained-step-input.ts | 60 +++++++++++-------- 2 files changed, 81 insertions(+), 26 deletions(-) diff --git a/packages/core/src/runtime/retained-step-input.test.ts b/packages/core/src/runtime/retained-step-input.test.ts index 58573edc22..482443844c 100644 --- a/packages/core/src/runtime/retained-step-input.test.ts +++ b/packages/core/src/runtime/retained-step-input.test.ts @@ -391,3 +391,50 @@ describe('checker execution surface', () => { expect(vm.runInContext('globalThis.__retainedTestCalls', context)).toBe(0); }); }); + +describe('walker primordials', () => { + it('uses captured primordials, not live host statics', () => { + const { context, workflowGlobal } = makeContext(); + const map = vm.runInContext('new Map([["k", { ok: true }]])', context); + + // If the walker consulted the live host statics, these would throw. + const originalDescriptor = Object.getOwnPropertyDescriptor; + const originalOwnKeys = Reflect.ownKeys; + (Object as any).getOwnPropertyDescriptor = () => { + throw new Error('live getOwnPropertyDescriptor used'); + }; + (Reflect as any).ownKeys = () => { + throw new Error('live ownKeys used'); + }; + try { + expect(isRetainedSerializationPassive(map, workflowGlobal)).toBe(true); + } finally { + (Object as any).getOwnPropertyDescriptor = originalDescriptor; + (Reflect as any).ownKeys = originalOwnKeys; + } + }); + + it('declines when host Function.prototype carries serializer statics', () => { + const { context, workflowGlobal } = makeContext(); + const plain = vm.runInContext('({ ok: true })', context); + expect(isRetainedSerializationPassive(plain, workflowGlobal)).toBe(true); + + Object.defineProperty( + Function.prototype, + Symbol.for('workflow-serialize'), + { + get() { + return undefined; + }, + configurable: true, + } + ); + try { + expect(isRetainedSerializationPassive(plain, workflowGlobal)).toBe(false); + } finally { + delete (Function.prototype as any)[Symbol.for('workflow-serialize')]; + } + + expect(isRetainedSerializationPassive(plain, workflowGlobal)).toBe(true); + }); +}); diff --git a/packages/core/src/runtime/retained-step-input.ts b/packages/core/src/runtime/retained-step-input.ts index 0da2ed82e3..e18a5db19e 100644 --- a/packages/core/src/runtime/retained-step-input.ts +++ b/packages/core/src/runtime/retained-step-input.ts @@ -48,8 +48,16 @@ const HOST_DISPATCH_CONSTRUCTORS = [ // Host primordials captured at module load — before any workflow code can // exist in the process — so the checker itself never invokes a live host -// method workflow code could have replaced (host constructors are reachable -// via e.g. `structuredClone(new Map()).constructor`). +// member workflow code could have replaced (host constructors are reachable +// via e.g. `structuredClone(new Map()).constructor` or exposed classes). +const hostGetOwnPropertyDescriptor = Object.getOwnPropertyDescriptor; +const hostGetPrototypeOf = Object.getPrototypeOf; +const hostIsFrozen = Object.isFrozen; +const hostOwnKeys = Reflect.ownKeys; +const hostIsArray = Array.isArray; +const hostIsInteger = Number.isInteger; +const hostNumber = Number; +const hostString = String; const hostMapForEach = Map.prototype.forEach; const hostSetForEach = Set.prototype.forEach; // biome-ignore lint/style/noNonNullAssertion: the %TypedArray% buffer getter always exists @@ -74,7 +82,7 @@ const TYPED_ARRAY_CONSTRUCTORS = [ ] as const; function hasOwn(target: object, key: string | symbol): boolean { - return Object.getOwnPropertyDescriptor(target, key) !== undefined; + return hostGetOwnPropertyDescriptor(target, key) !== undefined; } function isHostDispatchPristine(): boolean { @@ -84,11 +92,11 @@ function isHostDispatchPristine(): boolean { if (constructor === undefined) continue; if (hasOwn(constructor as object, Symbol.hasInstance)) return false; } - for (const constructor of [Object, Array]) { - if ( - hasOwn(constructor, WORKFLOW_SERIALIZE) || - hasOwn(constructor, 'classId') - ) { + // The class reducer reads `cls[WORKFLOW_SERIALIZE]` / `cls.classId` as + // inherited Gets, so the constructors' whole prototype chains — host + // Function.prototype and Object.prototype — must be clean too. + for (const target of [Object, Array, Function.prototype, Object.prototype]) { + if (hasOwn(target, WORKFLOW_SERIALIZE) || hasOwn(target, 'classId')) { return false; } } @@ -114,7 +122,7 @@ function isHostDispatchPristine(): boolean { // can redefine its globals (or their `prototype` slots) with accessors, and // validation must not execute workflow-owned code. function ownDataProperty(target: object, key: string): unknown { - const descriptor = Object.getOwnPropertyDescriptor(target, key); + const descriptor = hostGetOwnPropertyDescriptor(target, key); return descriptor && 'value' in descriptor ? descriptor.value : undefined; } @@ -150,7 +158,7 @@ function frozenRealmPrototype( constructorName: string ): object | undefined { const prototype = constructorPrototype(workflowGlobal, constructorName); - return prototype !== undefined && Object.isFrozen(prototype) + return prototype !== undefined && hostIsFrozen(prototype) ? prototype : undefined; } @@ -160,7 +168,7 @@ function hasAllowedPrototype( workflowGlobal: Record, constructorName: string ): boolean { - const prototype = Object.getPrototypeOf(value); + const prototype = hostGetPrototypeOf(value); return ( prototype === constructorPrototype( @@ -171,12 +179,12 @@ function hasAllowedPrototype( } function isArrayIndex(key: string): boolean { - const index = Number(key); + const index = hostNumber(key); return ( - Number.isInteger(index) && + hostIsInteger(index) && index >= 0 && index < 2 ** 32 - 1 && - String(index) === key + hostString(index) === key ); } @@ -187,7 +195,7 @@ function isPassiveArrayProperty( seen: WeakSet ): boolean { if (key === 'length') return true; - const descriptor = Object.getOwnPropertyDescriptor(array, key); + const descriptor = hostGetOwnPropertyDescriptor(array, key); if (!descriptor) return false; // Only own enumerable data indices are passive. Anything hidden — symbol // tags, non-enumerable properties, accessors — can be observed by @@ -209,7 +217,7 @@ function isPassiveArray( ): boolean { return ( hasAllowedPrototype(value, workflowGlobal, 'Array') && - Reflect.ownKeys(value).every((key) => + hostOwnKeys(value).every((key) => isPassiveArrayProperty(value, key, workflowGlobal, seen) ) ); @@ -220,7 +228,7 @@ function isPassiveArray( // symbol could still be observed through other dispatch lookups, and clean // instances are the overwhelmingly common case anyway. function hasNoOwnProperties(value: object): boolean { - return Reflect.ownKeys(value).length === 0; + return hostOwnKeys(value).length === 0; } function isPassiveMap( @@ -228,7 +236,7 @@ function isPassiveMap( workflowGlobal: Record, seen: WeakSet ): boolean { - const prototype = Object.getPrototypeOf(value); + const prototype = hostGetPrototypeOf(value); if (prototype !== frozenRealmPrototype(workflowGlobal, 'Map')) return false; if (!hasNoOwnProperties(value)) return false; let passive = true; @@ -247,7 +255,7 @@ function isPassiveSet( workflowGlobal: Record, seen: WeakSet ): boolean { - const prototype = Object.getPrototypeOf(value); + const prototype = hostGetPrototypeOf(value); if (prototype !== frozenRealmPrototype(workflowGlobal, 'Set')) return false; if (!hasNoOwnProperties(value)) return false; let passive = true; @@ -261,7 +269,7 @@ function isPassiveDate( value: object, workflowGlobal: Record ): boolean { - const prototype = Object.getPrototypeOf(value); + const prototype = hostGetPrototypeOf(value); return ( prototype === frozenRealmPrototype(workflowGlobal, 'Date') && hasNoOwnProperties(value) @@ -276,7 +284,7 @@ function isPassiveTypedArray( // prototypes — NOT "anything chaining to %TypedArray%": a workflow can // manufacture a frozen hostile prototype with a delegating `buffer` getter // and setPrototypeOf a real typed array onto it. - const prototype = Object.getPrototypeOf(value); + const prototype = hostGetPrototypeOf(value); if ( !TYPED_ARRAY_CONSTRUCTORS.some( (name) => prototype === frozenRealmPrototype(workflowGlobal, name) @@ -286,7 +294,7 @@ function isPassiveTypedArray( } // Own keys on a typed array are exactly its canonical indices. if ( - !Reflect.ownKeys(value).every( + !hostOwnKeys(value).every( (key) => typeof key === 'string' && isArrayIndex(key) ) ) { @@ -301,7 +309,7 @@ function isPassiveArrayBuffer( value: object, workflowGlobal: Record ): boolean { - const prototype = Object.getPrototypeOf(value); + const prototype = hostGetPrototypeOf(value); return ( prototype === frozenRealmPrototype(workflowGlobal, 'ArrayBuffer') && hasNoOwnProperties(value) @@ -314,7 +322,7 @@ function isPassiveObjectProperty( workflowGlobal: Record, seen: WeakSet ): boolean { - const descriptor = Object.getOwnPropertyDescriptor(object, key); + const descriptor = hostGetOwnPropertyDescriptor(object, key); if (!descriptor) return false; // Only own enumerable string-keyed data properties are passive. Anything // hidden — symbol tags, non-enumerable properties, accessors — can be @@ -335,7 +343,7 @@ function isPassivePlainObject( seen: WeakSet ): boolean { if (!hasAllowedPrototype(value, workflowGlobal, 'Object')) return false; - return Reflect.ownKeys(value).every((key) => + return hostOwnKeys(value).every((key) => isPassiveObjectProperty(value, key, workflowGlobal, seen) ); } @@ -378,7 +386,7 @@ function isPassive( if (seen.has(value)) return true; seen.add(value); - if (Array.isArray(value)) return isPassiveArray(value, workflowGlobal, seen); + if (hostIsArray(value)) return isPassiveArray(value, workflowGlobal, seen); if (types.isMap(value)) return isPassiveMap(value, workflowGlobal, seen); if (types.isSet(value)) return isPassiveSet(value, workflowGlobal, seen); if (types.isDate(value)) return isPassiveDate(value, workflowGlobal); From 59e21d64596770f6487f0c656415b04636a9211b Mon Sep 17 00:00:00 2001 From: Nathan Colosimo <110621881+NathanColosimo@users.noreply.github.com> Date: Tue, 21 Jul 2026 16:41:00 -0700 Subject: [PATCH 27/43] fix(core): stale-suspension generation token; cover BigInt toString MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - Suspension signals capture ctx.suspensionGeneration when scheduled and no-op if the session resumed past that boundary. The harmful interleaving was already unreachable (queue items are deleted on consume, completion writes state synchronously, nextTick precedes timers) — the token turns those ordering facts into an explicit invariant. - The BigInt reducer calls .toString() on primitives from host code, which resolves on host BigInt.prototype: its identity joins the host dispatch check, and the VM BigInt.prototype is frozen besides. --- packages/core/src/private.ts | 7 +++++++ .../src/runtime/retained-step-input.test.ts | 21 +++++++++++++++++++ .../core/src/runtime/retained-step-input.ts | 7 +++++++ packages/core/src/step.ts | 4 ++++ packages/core/src/vm/index.ts | 1 + packages/core/src/workflow.ts | 2 ++ 6 files changed, 42 insertions(+) diff --git a/packages/core/src/private.ts b/packages/core/src/private.ts index 36072b290c..96815fa4e5 100644 --- a/packages/core/src/private.ts +++ b/packages/core/src/private.ts @@ -133,6 +133,13 @@ export interface WorkflowOrchestratorContext { runId: string; encryptionKey: CryptoKey | undefined; globalThis: typeof globalThis; + /** + * Increments on every retained-session resume. Suspension signals capture + * it when scheduled and no-op if it moved — a timer queued at boundary N + * must not signal after the session resumed into boundary N+1, so the + * state machine never has to reason about cross-boundary timer ordering. + */ + suspensionGeneration: number; eventsConsumer: EventsConsumer; /** * Map of pending invocations keyed by correlationId. diff --git a/packages/core/src/runtime/retained-step-input.test.ts b/packages/core/src/runtime/retained-step-input.test.ts index 482443844c..ce8eafa023 100644 --- a/packages/core/src/runtime/retained-step-input.test.ts +++ b/packages/core/src/runtime/retained-step-input.test.ts @@ -392,6 +392,27 @@ describe('checker execution surface', () => { }); }); +describe('bigint serialization', () => { + it('declines retention while host BigInt.prototype.toString is replaced', () => { + const { context, workflowGlobal } = makeContext(); + const value = vm.runInContext('({ big: 42n })', context); + expect(isRetainedSerializationPassive(value, workflowGlobal)).toBe(true); + + const original = BigInt.prototype.toString; + // biome-ignore lint/suspicious/noGlobalAssign: simulating workflow-realm tampering + BigInt.prototype.toString = function (this: bigint, ...args: [number?]) { + return original.apply(this, args); + }; + try { + expect(isRetainedSerializationPassive(value, workflowGlobal)).toBe(false); + } finally { + BigInt.prototype.toString = original; + } + + expect(isRetainedSerializationPassive(value, workflowGlobal)).toBe(true); + }); +}); + describe('walker primordials', () => { it('uses captured primordials, not live host statics', () => { const { context, workflowGlobal } = makeContext(); diff --git a/packages/core/src/runtime/retained-step-input.ts b/packages/core/src/runtime/retained-step-input.ts index e18a5db19e..7fec0156da 100644 --- a/packages/core/src/runtime/retained-step-input.ts +++ b/packages/core/src/runtime/retained-step-input.ts @@ -58,6 +58,7 @@ const hostIsArray = Array.isArray; const hostIsInteger = Number.isInteger; const hostNumber = Number; const hostString = String; +const hostBigIntToString = BigInt.prototype.toString; const hostMapForEach = Map.prototype.forEach; const hostSetForEach = Set.prototype.forEach; // biome-ignore lint/style/noNonNullAssertion: the %TypedArray% buffer getter always exists @@ -115,6 +116,12 @@ function isHostDispatchPristine(): boolean { // added @@hasInstance would be found by dispatch lookup. (Host // Function.prototype's @@hasInstance is spec non-configurable.) if (hasOwn(Object.prototype, Symbol.hasInstance)) return false; + // The BigInt reducer calls `.toString()` on bigint primitives from host + // code, which resolves on the HOST BigInt.prototype (primitives are + // realm-less; method lookup uses the running code's realm). + if (ownDataProperty(BigInt.prototype, 'toString') !== hostBigIntToString) { + return false; + } return true; } diff --git a/packages/core/src/step.ts b/packages/core/src/step.ts index 1b14165b91..40d20bc914 100644 --- a/packages/core/src/step.ts +++ b/packages/core/src/step.ts @@ -56,7 +56,11 @@ export function createUseStep(ctx: WorkflowOrchestratorContext) { // Crucially, if we got here, then this step Promise does // not resolve so that the user workflow code does not proceed any further. // Notify the workflow handler that this step has not been run / has not completed yet. + const generation = ctx.suspensionGeneration; scheduleWhenIdle(ctx, () => { + // A retained session may have resumed past this boundary while + // the timer was queued; a stale signal must not fire. + if (generation !== ctx.suspensionGeneration) return; ctx.onWorkflowError( new WorkflowSuspension(ctx.invocationsQueue, ctx.globalThis) ); diff --git a/packages/core/src/vm/index.ts b/packages/core/src/vm/index.ts index 376fd01961..0d32faad6d 100644 --- a/packages/core/src/vm/index.ts +++ b/packages/core/src/vm/index.ts @@ -173,6 +173,7 @@ export function freezeSerializationIntrinsics(g: typeof globalThis): void { if (constructor !== undefined) Object.freeze(constructor.prototype); } Object.freeze(g.ArrayBuffer.prototype); + Object.freeze(g.BigInt.prototype); const hostGlobal = globalThis as unknown as Record; for (const name of SERIALIZATION_BINDINGS) { const descriptor = Object.getOwnPropertyDescriptor(g, name); diff --git a/packages/core/src/workflow.ts b/packages/core/src/workflow.ts index 83a8154ca0..70beca73fd 100644 --- a/packages/core/src/workflow.ts +++ b/packages/core/src/workflow.ts @@ -452,6 +452,7 @@ function createWorkflowSession({ promiseQueueHolder.current = value; }, pendingDeliveries: 0, + suspensionGeneration: 0, pendingDeliveryBarriers: new Map(), replayPayloadCache, }; @@ -1155,6 +1156,7 @@ function createWorkflowSession({ } const interruption = withResolvers(); state = { type: 'running', interruption }; + workflowContext.suspensionGeneration++; eventsConsumer.append(nextEvents.slice(consumedEvents.length)); return { type: 'resumed', From b908241580c8f66bc98507cb58656944e5f469b3 Mon Sep 17 00:00:00 2001 From: Nathan Colosimo <110621881+NathanColosimo@users.noreply.github.com> Date: Tue, 21 Jul 2026 19:32:12 -0700 Subject: [PATCH 28/43] feat(core): deterministic sandbox hardening MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - crypto.subtle.digest computes synchronously via node:crypto: byte-identical values, promise settles on a deterministic microtask, full BufferSource validation (internal-slot view reads, SAB rejection) - Atomics.waitAsync (a wall-clock timer), async WebAssembly compilation, WeakRef, and FinalizationRegistry are removed from the sandbox — wall clock and GC observation are unreplayable; sync WebAssembly constructors remain - freezeSerializationIntrinsics pins the universal dispatch surfaces: Object.prototype/Array.prototype/Function.prototype are frozen (every missed property read and hasInstance lookup terminates there) and serialization-referenced global bindings are non-writable. Value-type prototypes and constructor statics stay patchable so polyfills (Temporal's Date.prototype.toTemporalInstant, core-js Set.prototype .union / Object.groupBy) keep working — the retained-input gate verifies the members serialization executes per boundary instead. Groundwork for retained-VM replay (#2990). --- .changeset/vm-determinism-hardening.md | 6 + .../v5/api-reference/workflow-globals.mdx | 6 +- packages/core/src/vm/index.test.ts | 221 +++++++++++++++++- packages/core/src/vm/index.ts | 199 +++++++++++++++- packages/core/src/workflow.ts | 6 +- 5 files changed, 433 insertions(+), 5 deletions(-) create mode 100644 .changeset/vm-determinism-hardening.md diff --git a/.changeset/vm-determinism-hardening.md b/.changeset/vm-determinism-hardening.md new file mode 100644 index 0000000000..d40a2cffd8 --- /dev/null +++ b/.changeset/vm-determinism-hardening.md @@ -0,0 +1,6 @@ +--- +'@workflow/core': patch +'workflow': patch +--- + +Deterministic sandbox hardening: `crypto.subtle.digest` in workflow functions now computes synchronously via `node:crypto` (byte-identical values, deterministic timing under replay); `Atomics.waitAsync`, async `WebAssembly` compilation, `WeakRef`, and `FinalizationRegistry` are no longer exposed (wall-clock timing and GC observation cannot be replayed); and the sandbox pins the surfaces serialization dispatch resolves through: `Object.prototype`/`Array.prototype`/`Function.prototype` are frozen and serialization-relevant global bindings are non-writable. Built-in prototypes and constructor statics remain patchable, so polyfills (Temporal, core-js) keep working. diff --git a/docs/content/docs/v5/api-reference/workflow-globals.mdx b/docs/content/docs/v5/api-reference/workflow-globals.mdx index 6ec32b6b0b..2ec290738f 100644 --- a/docs/content/docs/v5/api-reference/workflow-globals.mdx +++ b/docs/content/docs/v5/api-reference/workflow-globals.mdx @@ -26,7 +26,7 @@ These APIs are available but are **seeded or fixed** to ensure deterministic beh | [`Date`](https://developer.mozilla.org/en-US/docs/Web/JavaScript/Reference/Global_Objects/Date) / `Date.now()` / `new Date()` | Returns a fixed timestamp that advances with the workflow's logical clock | | [`crypto.getRandomValues()`](https://developer.mozilla.org/en-US/docs/Web/API/Crypto/getRandomValues) | Seeded — produces deterministic output for a given workflow run | | [`crypto.randomUUID()`](https://developer.mozilla.org/en-US/docs/Web/API/Crypto/randomUUID) | Seeded — produces deterministic UUIDs for a given workflow run | -| [`crypto.subtle.digest()`](https://developer.mozilla.org/en-US/docs/Web/API/SubtleCrypto/digest) | Passes through to the real implementation (SHA-256, etc. are deterministic by nature) | +| [`crypto.subtle.digest()`](https://developer.mozilla.org/en-US/docs/Web/API/SubtleCrypto/digest) | Computed synchronously via `node:crypto` (values are byte-identical to WebCrypto), so the promise settles at a deterministic point during replay | You can safely use `Math.random()`, `Date.now()`, and `crypto.randomUUID()` in workflow functions. The framework ensures these return the same values across replays. @@ -100,3 +100,7 @@ The following are **not available** in workflow functions. Move this logic to [s - **Global `fetch`**: Use [`import { fetch } from "workflow"`](/docs/api-reference/workflow/fetch) instead. See [fetch-in-workflow](/docs/errors/fetch-in-workflow). - **Timers**: `setTimeout`, `setInterval`, `setImmediate`, and their `clear*` counterparts. Use [`sleep()`](/docs/api-reference/workflow/sleep) instead. See [timeout-in-workflow](/docs/errors/timeout-in-workflow). - **`Buffer`**: Node.js-specific API. Use `Uint8Array` with `toBase64()` / `fromBase64()` / `toHex()` / `fromHex()` for binary data encoding, or `atob()` / `btoa()` for string-based base64. +- **`WeakRef` and `FinalizationRegistry`**: garbage-collection timing is not deterministic, so observing it would make workflow code impossible to replay. (`WeakMap` and `WeakSet` remain available — they do not expose GC state.) +- **`Atomics.waitAsync`**: a wall-clock timer, which cannot be replayed. Use [`sleep()`](/docs/api-reference/workflow/sleep) instead. +- **Async `WebAssembly` compilation** (`compile`, `instantiate`, `compileStreaming`, `instantiateStreaming`): resolves on compile-thread timing. The synchronous `new WebAssembly.Module()` and `new WebAssembly.Instance()` constructors remain available. +- **Mutating core intrinsics**: `Object.prototype`, `Array.prototype`, and `Function.prototype` are frozen, and the global bindings for serialization-relevant built-ins (`Map`, `Date`, `Headers`, and similar) cannot be reassigned — serialization dispatch resolves through these, so redefining them would make replay unfaithful. Adding methods to built-in prototypes (e.g. a Temporal polyfill's `Date.prototype.toTemporalInstant`) and polyfilling constructor statics (`Object.groupBy`) remain allowed. diff --git a/packages/core/src/vm/index.test.ts b/packages/core/src/vm/index.test.ts index 0381b60f63..f28f330cb4 100644 --- a/packages/core/src/vm/index.test.ts +++ b/packages/core/src/vm/index.test.ts @@ -1,6 +1,6 @@ import * as vm from 'node:vm'; import { describe, expect, it } from 'vitest'; -import { createContext } from './index.js'; +import { createContext, freezeSerializationIntrinsics } from './index.js'; const seed = 'entropy seed'; const fixedTimestamp = 1234567890000; @@ -257,3 +257,222 @@ describe('createContext', () => { expect(result).toBe('undefined'); }); }); + +describe('host-timed async', () => { + it('does not expose any API that settles on host timing', () => { + const { context } = createContext({ seed, fixedTimestamp }); + + expect(vm.runInContext('typeof Atomics.waitAsync', context)).toBe( + 'undefined' + ); + expect(vm.runInContext('typeof WebAssembly.compile', context)).toBe( + 'undefined' + ); + expect(vm.runInContext('typeof WebAssembly.instantiate', context)).toBe( + 'undefined' + ); + expect( + vm.runInContext('typeof WebAssembly.compileStreaming', context) + ).toBe('undefined'); + expect( + vm.runInContext('typeof WebAssembly.instantiateStreaming', context) + ).toBe('undefined'); + // Deterministic synchronous WebAssembly stays available. + expect(vm.runInContext('typeof WebAssembly.Module', context)).toBe( + 'function' + ); + }); +}); + +describe('crypto.subtle.digest', () => { + it.each([ + 'SHA-1', + 'SHA-256', + 'SHA-384', + 'SHA-512', + ])('matches WebCrypto for %s', async (algorithm) => { + const { context } = createContext({ seed, fixedTimestamp }); + + const result = await vm.runInContext( + `crypto.subtle.digest(${JSON.stringify(algorithm)}, new Uint8Array([1,2,3,4,5,255,0,128]))`, + context + ); + const expected = await globalThis.crypto.subtle.digest( + algorithm, + new Uint8Array([1, 2, 3, 4, 5, 255, 0, 128]) + ); + expect(result).toBeInstanceOf(ArrayBuffer); + expect(new Uint8Array(result)).toEqual(new Uint8Array(expected)); + }); + + it('accepts the { name } algorithm form and ArrayBuffer input', async () => { + const { context } = createContext({ seed, fixedTimestamp }); + + const result = await vm.runInContext( + 'crypto.subtle.digest({ name: "sha-256" }, new ArrayBuffer(4))', + context + ); + const expected = await globalThis.crypto.subtle.digest( + 'SHA-256', + new ArrayBuffer(4) + ); + expect(new Uint8Array(result)).toEqual(new Uint8Array(expected)); + }); + + it('respects typed-array subviews', async () => { + const { context } = createContext({ seed, fixedTimestamp }); + + const result = await vm.runInContext( + 'const bytes = new Uint8Array([9, 1, 2, 9]); crypto.subtle.digest("SHA-256", bytes.subarray(1, 3))', + context + ); + const expected = await globalThis.crypto.subtle.digest( + 'SHA-256', + new Uint8Array([1, 2]) + ); + expect(new Uint8Array(result)).toEqual(new Uint8Array(expected)); + }); + + it('rejects unsupported algorithms like WebCrypto does', async () => { + const { context } = createContext({ seed, fixedTimestamp }); + + await expect( + vm.runInContext('crypto.subtle.digest("MD5", new Uint8Array(1))', context) + ).rejects.toMatchObject({ name: 'NotSupportedError' }); + }); +}); + +describe('GC observation', () => { + it('does not expose WeakRef or FinalizationRegistry', () => { + const { context } = createContext({ seed, fixedTimestamp }); + + expect(vm.runInContext('typeof WeakRef', context)).toBe('undefined'); + expect(vm.runInContext('typeof FinalizationRegistry', context)).toBe( + 'undefined' + ); + // WeakMap/WeakSet do not expose GC state and stay available. + expect(vm.runInContext('typeof WeakMap', context)).toBe('function'); + expect(vm.runInContext('typeof WeakSet', context)).toBe('function'); + }); +}); + +describe('crypto.subtle.digest input validation', () => { + it.each([ + ['a number', '2000000000'], + ['a plain object', '({})'], + ['a string', '"data"'], + ['null', 'null'], + ])('rejects %s with TypeError', async (_label, expression) => { + const { context } = createContext({ seed, fixedTimestamp }); + + await expect( + vm.runInContext(`crypto.subtle.digest("SHA-256", ${expression})`, context) + ).rejects.toThrow(TypeError); + }); +}); + +describe('crypto.subtle.digest SharedArrayBuffer rejection', () => { + it('rejects SharedArrayBuffer-backed views like WebCrypto does', async () => { + const { context } = createContext({ seed, fixedTimestamp }); + + await expect( + vm.runInContext( + 'crypto.subtle.digest("SHA-256", new Uint8Array(new SharedArrayBuffer(4)))', + context + ) + ).rejects.toThrow(TypeError); + }); +}); + +describe('crypto.subtle.digest view metadata', () => { + it('reads view ranges from internal slots, ignoring shadowed properties', async () => { + const { context } = createContext({ seed, fixedTimestamp }); + + const result = await vm.runInContext( + `(() => { + const view = new Uint8Array([1, 2, 3, 4]); + Object.defineProperty(view, "byteLength", { value: 0 }); + Object.defineProperty(view, "byteOffset", { value: 2 }); + return crypto.subtle.digest("SHA-256", view); + })()`, + context + ); + const expected = await globalThis.crypto.subtle.digest( + 'SHA-256', + new Uint8Array([1, 2, 3, 4]) + ); + expect(new Uint8Array(result)).toEqual(new Uint8Array(expected)); + }); +}); + +describe('freezeSerializationIntrinsics', () => { + it('freezes the universal lookup backstops', () => { + const { context, globalThis: g } = createContext({ seed, fixedTimestamp }); + freezeSerializationIntrinsics(g); + + expect(() => + vm.runInContext(`"use strict"; Object.prototype.polluted = 1`, context) + ).toThrow(/not extensible/); + expect(() => + vm.runInContext( + `"use strict"; Object.defineProperty(Array.prototype, "signal", { get() { return 1; } })`, + context + ) + ).toThrow(/not extensible/); + expect(() => + vm.runInContext( + `"use strict"; Function.prototype[Symbol.hasInstance] = () => true`, + context + ) + ).toThrow(/read only|not extensible/); + }); + + it('pins global bindings, including intentionally absent ones', () => { + const { context, globalThis: g } = createContext({ seed, fixedTimestamp }); + freezeSerializationIntrinsics(g); + + vm.runInContext( + 'try { globalThis.Map = function () {} } catch {}', + context + ); + expect(vm.runInContext('typeof Map.prototype.get', context)).toBe( + 'function' + ); + vm.runInContext( + 'try { globalThis.Request = function () {} } catch {}', + context + ); + expect(vm.runInContext('typeof Request', context)).toBe('undefined'); + }); + + it('leaves value-type prototypes and constructor statics patchable (polyfills)', () => { + const { context, globalThis: g } = createContext({ seed, fixedTimestamp }); + freezeSerializationIntrinsics(g); + + // Temporal-style prototype method and modern static polyfills must work. + const result = vm.runInContext( + `"use strict"; + Date.prototype.toTemporalInstant = function () { return "instant:" + this.getTime(); }; + Object.groupBy = (items, fn) => items.reduce((acc, x) => ((acc[fn(x)] ??= []).push(x), acc), {}); + Set.prototype.union = function (other) { return new Set([...this, ...other]); }; + [ + new Date(5).toTemporalInstant(), + Object.groupBy([1, 2, 3], (x) => (x % 2 ? "odd" : "even")).odd.length, + new Set([1]).union(new Set([2])).size, + ]`, + context + ); + expect(result[0]).toBe('instant:5'); + expect(result[1]).toBe(2); + expect(result[2]).toBe(2); + }); + + it('leaves ordinary workflow globals writable', () => { + const { context, globalThis: g } = createContext({ seed, fixedTimestamp }); + freezeSerializationIntrinsics(g); + + expect( + vm.runInContext('globalThis.myState = { ok: true }; myState.ok', context) + ).toBe(true); + }); +}); diff --git a/packages/core/src/vm/index.ts b/packages/core/src/vm/index.ts index c0003bdc39..f23bb71488 100644 --- a/packages/core/src/vm/index.ts +++ b/packages/core/src/vm/index.ts @@ -1,3 +1,5 @@ +import { createHash } from 'node:crypto'; +import { types } from 'node:util'; import { runInContext, createContext as vmCreateContext } from 'node:vm'; import { WorkflowRuntimeError } from '@workflow/errors'; import seedrandom from 'seedrandom'; @@ -10,6 +12,153 @@ export interface CreateContextOptions { fixedTimestamp: number; } +// WebCrypto digest algorithm names → node:crypto (OpenSSL) names. +const DIGEST_ALGORITHMS: Record = { + 'SHA-1': 'sha1', + 'SHA-256': 'sha256', + 'SHA-384': 'sha384', + 'SHA-512': 'sha512', +}; + +// Intrinsic prototype getters, captured so view metadata is read from +// internal slots like WebCrypto's BufferSource conversion — own properties +// shadowing `buffer`/`byteOffset`/`byteLength` on a view must not change +// which bytes are hashed. +function intrinsicGetter(prototype: object, name: string) { + // biome-ignore lint/style/noNonNullAssertion: intrinsic accessors always exist + return Object.getOwnPropertyDescriptor(prototype, name)!.get!; +} +const arrayBufferByteLength = intrinsicGetter( + ArrayBuffer.prototype, + 'byteLength' +); +const typedArrayPrototype = Object.getPrototypeOf( + Uint8Array.prototype +) as object; +const viewGetters = { + typedArray: { + buffer: intrinsicGetter(typedArrayPrototype, 'buffer'), + byteOffset: intrinsicGetter(typedArrayPrototype, 'byteOffset'), + byteLength: intrinsicGetter(typedArrayPrototype, 'byteLength'), + }, + dataView: { + buffer: intrinsicGetter(DataView.prototype, 'buffer'), + byteOffset: intrinsicGetter(DataView.prototype, 'byteOffset'), + byteLength: intrinsicGetter(DataView.prototype, 'byteLength'), + }, +}; + +// WebCrypto BufferSource conversion: typed-array/DataView views are read via +// internal slots; anything else must be a real ArrayBuffer (the native +// byteLength getter is a brand check that works across vm realms) so that +// e.g. a plain number is rejected with TypeError instead of allocating a +// Uint8Array of that length. +function toDigestBytes(data: ArrayBuffer | ArrayBufferView): Uint8Array { + if (types.isTypedArray(data) || types.isDataView(data)) { + const getters = types.isDataView(data) + ? viewGetters.dataView + : viewGetters.typedArray; + const buffer = getters.buffer.call(data) as ArrayBuffer; + // WebCrypto's BufferSource excludes SharedArrayBuffer-backed views. + if (types.isSharedArrayBuffer(buffer)) { + throw new TypeError( + 'crypto.subtle.digest does not accept SharedArrayBuffer-backed views' + ); + } + return new Uint8Array( + buffer, + getters.byteOffset.call(data) as number, + getters.byteLength.call(data) as number + ); + } + arrayBufferByteLength.call(data); + return new Uint8Array(data as ArrayBuffer); +} + +// Global bindings the serialization reducers dispatch on (`value instanceof +// global.X` — e.g. the Map/Set/Headers reducers in +// packages/core/src/serialization/reducers/common.ts:312-329 and the +// Request/Response/stream reducers in packages/core/src/serialization.ts). +// Pinned (non-writable) so workflow code cannot swap in a constructor whose +// `Symbol.hasInstance` the serializer would then execute. Absent bindings are +// pinned to `undefined` so a spoofing global cannot be introduced either. +const SERIALIZATION_BINDINGS = [ + 'Object', + 'Array', + 'Function', + 'Map', + 'Set', + 'Date', + 'RegExp', + 'ArrayBuffer', + 'SharedArrayBuffer', + 'DataView', + 'Int8Array', + 'Uint8Array', + 'Uint8ClampedArray', + 'Int16Array', + 'Uint16Array', + 'Int32Array', + 'Uint32Array', + 'Float16Array', + 'Float32Array', + 'Float64Array', + 'BigInt64Array', + 'BigUint64Array', + 'Headers', + 'URL', + 'URLSearchParams', + 'DOMException', + 'AbortController', + 'AbortSignal', + 'Request', + 'Response', + 'ReadableStream', + 'WritableStream', + 'TransformStream', +] as const; + +/** + * Pin the sandbox surfaces that serialization dispatch resolves through. + * Called after ALL SDK globals are installed, immediately before the + * workflow bundle evaluates. + * + * Frozen — the universal lookup backstops only: + * - `Object.prototype` / `Array.prototype`: every missed property read on a + * plain object or array terminates here (reducer probes like `.signal`, + * `Symbol.toStringTag` lookups by devalue's type detection, `constructor` + * reads by the class reducer in + * packages/core/src/serialization/reducers/class.ts:26-43), so a planted + * accessor here would execute during serialization of ANY value. + * - `Function.prototype`: `x instanceof C` resolves `Symbol.hasInstance` + * through the constructor's prototype chain (tc39.es/ecma262/#sec-instanceofoperator), + * which ends here for every constructor. + * No real-world polyfill patches these three, so freezing them costs nothing. + * + * NOT frozen — constructors and the value-type prototypes (`Map.prototype`, + * `Date.prototype`, typed arrays, …): polyfills legitimately add methods + * there (Temporal's `Date.prototype.toTemporalInstant`, core-js + * `Set.prototype.union`, `Object.groupBy`, `Array.fromAsync`). New + * data-valued members are inert during serialization — only the specific + * members the serializer EXECUTES matter, and the retained-input gate + * verifies those per boundary against captured originals instead + * (see runtime/retained-step-input.ts). + */ +export function freezeSerializationIntrinsics(g: typeof globalThis): void { + Object.freeze(g.Object.prototype); + Object.freeze(g.Array.prototype); + Object.freeze(g.Function.prototype); + for (const name of SERIALIZATION_BINDINGS) { + const descriptor = Object.getOwnPropertyDescriptor(g, name); + Object.defineProperty(g, name, { + value: descriptor && 'value' in descriptor ? descriptor.value : undefined, + writable: false, + configurable: false, + enumerable: descriptor?.enumerable ?? false, + }); + } +} + /** * Creates a Node.js `vm.Context` configured to be usable for * executing workflow logic in a deterministic environment. @@ -58,7 +207,53 @@ export function createContext(options: CreateContextOptions) { const randomUUID = createRandomUUID(rng); - const boundDigest = originalSubtle.digest.bind(originalSubtle); + // The sandbox must not expose any way for workflow code to observe host + // timing or host state: after this block, every promise a workflow can + // create settles either from the event log or within its own microtask + // cascade, so a suspended VM is fully quiescent and can be retained across + // inline steps (see `canRetainWorkflowSession`). + // + // - `Atomics.waitAsync` is a wall-clock timer (via SharedArrayBuffer). + // - The async `WebAssembly` entry points resolve on compile-thread timing; + // the synchronous `new WebAssembly.Module()` / `Instance()` remain. + // - `WeakRef.deref()` and finalizer callbacks observe GC timing. + // - Dynamic `import()` settles within a microtask (rejected: no + // `importModuleDynamically`), so it needs no handling. + const intrinsics = g as unknown as Record>; + delete intrinsics.Atomics.waitAsync; + delete intrinsics.WebAssembly.compile; + delete intrinsics.WebAssembly.instantiate; + delete intrinsics.WebAssembly.compileStreaming; + delete intrinsics.WebAssembly.instantiateStreaming; + delete intrinsics.WeakRef; + delete intrinsics.FinalizationRegistry; + + // `crypto.subtle.digest` computes synchronously via node:crypto, so its + // promise settles on a deterministic microtask instead of host threadpool + // timing — a digest can never advance a suspended workflow, and + // digest-using VMs stay retainable. Values are byte-identical to WebCrypto. + const digest = ( + algorithm: string | { name: string }, + data: ArrayBuffer | ArrayBufferView + ): Promise => { + try { + const name = typeof algorithm === 'string' ? algorithm : algorithm.name; + const ossl = DIGEST_ALGORITHMS[name.toUpperCase()]; + if (!ossl) { + throw new DOMException( + `Unrecognized algorithm name: ${name}`, + 'NotSupportedError' + ); + } + const hash = createHash(ossl).update(toDigestBytes(data)).digest(); + // Copy out: small Buffers share the internal pool allocation. + const out = new ArrayBuffer(hash.byteLength); + new Uint8Array(out).set(hash); + return Promise.resolve(out); + } catch (error) { + return Promise.reject(error); + } + }; g.crypto = new Proxy(originalCrypto, { get(target, prop) { @@ -78,7 +273,7 @@ export function createContext(options: CreateContextOptions) { ); }; } else if (prop === 'digest') { - return boundDigest; + return digest; } return target[prop as keyof typeof originalSubtle]; }, diff --git a/packages/core/src/workflow.ts b/packages/core/src/workflow.ts index 06fe1d79e2..8ed18de3d9 100644 --- a/packages/core/src/workflow.ts +++ b/packages/core/src/workflow.ts @@ -38,7 +38,7 @@ import { import * as Attribute from './telemetry/semantic-conventions.js'; import { trace } from './telemetry.js'; import { getWorkflowRunStreamId } from './util.js'; -import { createContext } from './vm/index.js'; +import { createContext, freezeSerializationIntrinsics } from './vm/index.js'; import { runCachedWorkflowScript } from './vm/script-cache.js'; import { createAbortSignalStatics, @@ -847,6 +847,10 @@ export async function runWorkflow( // Script rather than the line just past the end of the bundle. That path // is rare (it requires the lookup `?.get(...)` expression to throw) and // does not affect the workflow function or replay determinism. + // All SDK globals are installed; pin the serialization-consulted + // intrinsics before any workflow code can run. + freezeSerializationIntrinsics(vmGlobalThis); + runCachedWorkflowScript(workflowCode, filename, context); const workflowFn = runCachedWorkflowScript( `globalThis.__private_workflows?.get(${JSON.stringify(workflowRun.workflowName)})`, From bb2098ab347ce66a9bac129e0785f24bbdef8a36 Mon Sep 17 00:00:00 2001 From: Nathan Colosimo <110621881+NathanColosimo@users.noreply.github.com> Date: Wed, 22 Jul 2026 10:47:00 -0700 Subject: [PATCH 29/43] feat(core): retain the workflow VM across inline steps (primitive args) Keeps the suspended workflow VM, its events consumer, and the paused async stack alive across inline step executions within one invocation. Each loop iteration appends only the newly written events instead of replaying the entire event log in a fresh VM, so step-to-step overhead stays flat as runs grow. - WorkflowSession wraps executeWorkflow: suspended sessions expose resume(events) which appends to the retained EventsConsumer and lets the parked run() continuation settle; any divergence (unexpected suspension shape, consumer error) demotes to full replay permanently - Retention is gated per boundary: only suspensions whose queued step inputs are all primitives (null/undefined/boolean/number/string) are retainable, because serializing primitives executes no workflow code; a follow-up widens this to plain data and standard built-ins - Suspensions with hooks, waits, or attributes always fall back - A suspension generation token invalidates stale timer callbacks from an abandoned suspension so they cannot advance a resumed VM - WORKFLOW_RETAINED_VM=0 kill switch; telemetry records workflow.execution.mode = replay | retained Part 2 of the retained-VM stack (#2990); requires the determinism hardening in part 1. --- .changeset/retain-workflow-vm.md | 2 +- .../docs/v5/configuration/runtime-tuning.mdx | 2 +- packages/core/src/retained-vm-loop.test.ts | 20 - .../src/runtime/retained-step-input.test.ts | 461 ----- .../core/src/runtime/retained-step-input.ts | 427 ----- .../core/src/runtime/suspension-handler.ts | 33 +- packages/core/src/workflow.test.ts | 62 + packages/core/src/workflow.ts | 1512 ++++++++--------- 8 files changed, 830 insertions(+), 1689 deletions(-) delete mode 100644 packages/core/src/runtime/retained-step-input.test.ts delete mode 100644 packages/core/src/runtime/retained-step-input.ts diff --git a/.changeset/retain-workflow-vm.md b/.changeset/retain-workflow-vm.md index 9db15ed117..25d38b0d57 100644 --- a/.changeset/retain-workflow-vm.md +++ b/.changeset/retain-workflow-vm.md @@ -3,4 +3,4 @@ 'workflow': patch --- -Retain workflow execution across passive-data inline steps within one invocation; boundaries whose argument serialization could execute workflow code fall back to replay, and `WORKFLOW_RETAINED_VM=0` disables retention. `crypto.subtle.digest` in workflow functions now computes synchronously via `node:crypto`, so digest timing is deterministic under replay. `WeakRef`, `FinalizationRegistry`, `Atomics.waitAsync`, and async `WebAssembly` compilation are no longer exposed in workflow functions (GC observation and wall-clock timing cannot be replayed), and the serialization-relevant intrinsics (`Object`/`Array`/`Function`, the prototypes of `Map`/`Set`/`Date`/typed arrays/`ArrayBuffer`, and reducer-referenced global bindings) are frozen in the workflow sandbox. +Retain workflow execution across inline steps within one invocation; `WORKFLOW_RETAINED_VM=0` disables retention. Boundaries whose step arguments are not primitive values fall back to ordinary replay. diff --git a/docs/content/docs/v5/configuration/runtime-tuning.mdx b/docs/content/docs/v5/configuration/runtime-tuning.mdx index 72a4679171..9007fb7c2a 100644 --- a/docs/content/docs/v5/configuration/runtime-tuning.mdx +++ b/docs/content/docs/v5/configuration/runtime-tuning.mdx @@ -81,7 +81,7 @@ For example, a workflow can run a 10-minute inline step even with `WORKFLOW_REPL - Default: enabled - Keeps the suspended workflow VM alive across inline steps within one invocation, so each iteration of the inline loop appends only the newly written events instead of replaying the whole event log in a fresh VM. - Suspensions involving hooks, waits, or attributes — and any replay divergence — always fall back to a full replay. -- Step inputs made of plain data (objects, arrays, primitives) and standard built-ins (`Map`, `Set`, `Date`, typed arrays, `ArrayBuffer`) remain retainable. Anything whose serialization could execute workflow code — accessors, proxies, functions, custom class serializers, `RegExp` — falls back to replay for that boundary. +- Step inputs of primitive values remain retainable; anything whose serialization could execute workflow code falls back to replay for that boundary. (Support for plain objects, arrays, and standard built-ins lands in a follow-up.) - Set `0` or `false` to replay from scratch in a fresh VM on every iteration. ### `WORKFLOW_INLINE_OWNERSHIP` diff --git a/packages/core/src/retained-vm-loop.test.ts b/packages/core/src/retained-vm-loop.test.ts index 2085b1edad..d24b59bcc2 100644 --- a/packages/core/src/retained-vm-loop.test.ts +++ b/packages/core/src/retained-vm-loop.test.ts @@ -100,17 +100,6 @@ const mixedBatchWorkflow = `const s1 = globalThis[Symbol.for("WORKFLOW_USE_STEP" } globalThis.__private_workflows = new Map([["workflow", workflow]]);`; -// Map/Date/typed-array arguments serialize through pinned prototype members -// (see vm/serialization-pins.ts), so these boundaries stay retainable. -const builtinArgsWorkflow = `const s1 = globalThis[Symbol.for("WORKFLOW_USE_STEP")]("r_s1"); - const s2 = globalThis[Symbol.for("WORKFLOW_USE_STEP")]("r_s2"); - async function workflow() { - const a = await s1({ index: new Map([["k", 1]]), when: new Date(1234) }); - const b = await s2(new Uint8Array([1, 2, 3])); - return a + b; - } - globalThis.__private_workflows = new Map([["workflow", workflow]]);`; - // `crypto.subtle.digest` computes synchronously via node:crypto, so a // digest-using VM stays quiescent at suspension and remains retainable. const digestWorkflow = `const s1 = globalThis[Symbol.for("WORKFLOW_USE_STEP")]("r_s1"); @@ -330,15 +319,6 @@ describe('retained VM through the inline replay loop', () => { expect(vmBuilds).toBeGreaterThan(1); }); - it('retains boundaries whose args are supported built-ins', async () => { - const { vmBuilds, output } = await drive( - 'wrun_retained_builtins', - builtinArgsWorkflow - ); - expect(output).toBeInstanceOf(Uint8Array); - expect(vmBuilds).toBe(1); - }); - it('retains a VM that used the synchronous crypto.subtle.digest', async () => { const { vmBuilds, output } = await drive( 'wrun_retained_digest', diff --git a/packages/core/src/runtime/retained-step-input.test.ts b/packages/core/src/runtime/retained-step-input.test.ts deleted file mode 100644 index ce8eafa023..0000000000 --- a/packages/core/src/runtime/retained-step-input.test.ts +++ /dev/null @@ -1,461 +0,0 @@ -import vm from 'node:vm'; -import { describe, expect, it } from 'vitest'; -import { dehydrateStepArguments } from '../serialization.js'; -import { createContext, freezeSerializationIntrinsics } from '../vm/index.js'; -import { isRetainedSerializationPassive } from './retained-step-input.js'; - -const seed = 'retained-step-input'; -const fixedTimestamp = 1_700_000_000_000; - -function makeContext({ freeze = true } = {}) { - const { context, globalThis: workflowGlobal } = createContext({ - seed, - fixedTimestamp, - }); - // Mimic the globals workflow.ts installs before any serialization happens - // (the stream/request reducers dispatch on them unguarded). - for (const name of [ - 'ReadableStream', - 'WritableStream', - 'TransformStream', - 'Request', - 'Response', - 'AbortController', - 'AbortSignal', - ]) { - if ((workflowGlobal as any)[name] === undefined) { - (workflowGlobal as any)[name] = (globalThis as any)[name]; - } - } - if (freeze) freezeSerializationIntrinsics(workflowGlobal); - return { context, workflowGlobal }; -} - -describe('isRetainedSerializationPassive', () => { - it('accepts plain cross-realm data', () => { - const { context, workflowGlobal } = makeContext(); - const value = vm.runInContext( - `({ - nested: [{ ok: true }, "text", 42n], - sparse: [1, , 3], - flag: false, - })`, - context - ); - - expect(isRetainedSerializationPassive(value, workflowGlobal)).toBe(true); - }); - - it('accepts the supported built-ins on a frozen realm', () => { - const { context, workflowGlobal } = makeContext(); - for (const expression of [ - 'new Map([["k", { ok: true }]])', - 'new Set([1, "two"])', - 'new Date(1234)', - 'new Uint8Array([1, 2, 3])', - 'new Float32Array([1.5])', - 'new ArrayBuffer(8)', - '({ when: new Date(0), bytes: new Uint8Array(2), index: new Map() })', - ]) { - const value = vm.runInContext(expression, context); - expect(isRetainedSerializationPassive(value, workflowGlobal)).toBe(true); - } - }); - - it('accepts host-realm plain data but not host-realm built-ins', () => { - const { workflowGlobal } = makeContext(); - // Hydrated step results are host-realm plain objects/arrays. - expect( - isRetainedSerializationPassive({ nested: [{ ok: true }] }, workflowGlobal) - ).toBe(true); - // Host built-in prototypes cannot be frozen (process-shared) and are - // reachable from workflow code, so host-realm instances decline. - expect( - isRetainedSerializationPassive(new Map([['k', 1]]), workflowGlobal) - ).toBe(false); - expect(isRetainedSerializationPassive(new Date(0), workflowGlobal)).toBe( - false - ); - }); - - it('declines types whose serialization surface is not frozen', () => { - const { context, workflowGlobal } = makeContext(); - for (const expression of [ - '/workflow/gi', - 'new DataView(new ArrayBuffer(8))', - 'new SharedArrayBuffer(8)', - 'new Uint8Array(new SharedArrayBuffer(4))', - 'new Error("boom")', - 'new (class Sub extends Map {})()', - 'Object.assign(new Map(), { expando: 1 })', - 'Object.create(null)', - ]) { - const value = vm.runInContext(expression, context); - expect(isRetainedSerializationPassive(value, workflowGlobal)).toBe(false); - } - }); - - it('declines accessors without invoking them', () => { - const { context, workflowGlobal } = makeContext(); - const value = vm.runInContext( - `(() => { - globalThis.__retainedTestCalls = 0; - return { get value() { globalThis.__retainedTestCalls++; return 1; } }; - })()`, - context - ); - - expect(isRetainedSerializationPassive(value, workflowGlobal)).toBe(false); - expect(vm.runInContext('globalThis.__retainedTestCalls', context)).toBe(0); - }); - - it('declines proxies without invoking their traps', () => { - const { context, workflowGlobal } = makeContext(); - const value = vm.runInContext( - `(() => { - globalThis.__retainedTestCalls = 0; - return new Proxy({ value: 1 }, { - ownKeys(target) { - globalThis.__retainedTestCalls++; - return Reflect.ownKeys(target); - } - }); - })()`, - context - ); - - expect(isRetainedSerializationPassive(value, workflowGlobal)).toBe(false); - expect(vm.runInContext('globalThis.__retainedTestCalls', context)).toBe(0); - }); - - it('declines custom class serializers without invoking them', () => { - const { context, workflowGlobal } = makeContext(); - const value = vm.runInContext( - `(() => { - globalThis.__retainedTestCalls = 0; - class Value { - static classId = "test/Value"; - static [Symbol.for("workflow-serialize")](instance) { - globalThis.__retainedTestCalls++; - return { value: instance.value }; - } - constructor(value) { this.value = value; } - } - return new Value(1); - })()`, - context - ); - - expect(isRetainedSerializationPassive(value, workflowGlobal)).toBe(false); - expect(vm.runInContext('globalThis.__retainedTestCalls', context)).toBe(0); - }); - - it('declines hidden own keys (symbols, non-enumerables, constructor)', () => { - const { context, workflowGlobal } = makeContext(); - for (const expression of [ - `(() => { - const tagged = { plain: true }; - Object.defineProperty(tagged, Symbol.for("WORKFLOW_ABORT_STREAM_NAME"), { - value: "abort-stream", enumerable: false, - }); - return tagged; - })()`, - `(() => { - const hidden = { plain: true }; - Object.defineProperty(hidden, "signal", { - get() { return { aborted: false }; }, enumerable: false, - }); - return hidden; - })()`, - `(() => { - const arr = [1]; - Object.defineProperty(arr, "constructor", { - value: class Fake { static classId = "fake"; }, enumerable: false, - }); - return arr; - })()`, - `(() => { - const arr = [1, 2]; - Object.defineProperty(arr, "0", { value: 7, enumerable: false }); - return arr; - })()`, - ]) { - const value = vm.runInContext(expression, context); - expect(isRetainedSerializationPassive(value, workflowGlobal)).toBe(false); - } - }); - - it('freezing makes prototype patches impossible, so retention persists', () => { - const { context, workflowGlobal } = makeContext(); - const map = vm.runInContext('new Map([["k", 1]])', context); - expect(isRetainedSerializationPassive(map, workflowGlobal)).toBe(true); - - // Redefining a member the serializer reads throws on the frozen prototype. - for (const attempt of [ - '"use strict"; Object.defineProperty(Map.prototype, "constructor", { get() { return 1; } })', - '"use strict"; Map.prototype[Symbol.iterator] = function () {};', - '"use strict"; Date.prototype.toISOString = function () { return "x"; };', - '"use strict"; Object.defineProperty(Float32Array.prototype, "buffer", { get() {} })', - ]) { - expect(() => vm.runInContext(attempt, context)).toThrow( - /not extensible|read only|Cannot redefine/ - ); - } - // Nothing changed, so the built-ins remain retainable. - expect(isRetainedSerializationPassive(map, workflowGlobal)).toBe(true); - }); - - it('declines built-ins when the realm was never frozen', () => { - const { context, workflowGlobal } = makeContext({ freeze: false }); - const map = vm.runInContext('new Map()', context); - expect(isRetainedSerializationPassive(map, workflowGlobal)).toBe(false); - // Plain data does not depend on the built-in prototypes. - const plain = vm.runInContext('({ ok: true })', context); - expect(isRetainedSerializationPassive(plain, workflowGlobal)).toBe(true); - }); - - it('declines retention while a host dispatch constructor is hooked', () => { - const { context, workflowGlobal } = makeContext(); - const plain = vm.runInContext('({ ok: true })', context); - expect(isRetainedSerializationPassive(plain, workflowGlobal)).toBe(true); - - Object.defineProperty(Headers, Symbol.hasInstance, { - value: () => false, - configurable: true, - }); - try { - expect(isRetainedSerializationPassive(plain, workflowGlobal)).toBe(false); - } finally { - delete (Headers as any)[Symbol.hasInstance]; - } - - expect(isRetainedSerializationPassive(plain, workflowGlobal)).toBe(true); - }); -}); - -describe('serialization touches only the frozen surface', () => { - // THE coupling test: retention is only sound if every prototype member - // `dehydrateStepArguments` executes for the supported built-ins lives on an - // object `freezeSerializationIntrinsics` freezes. Wrap every configurable - // member on the relevant prototypes (in an unfrozen realm) with a recorder - // and assert the serializer hits nothing beyond this measured set — every - // entry of which is on a frozen prototype in production. If serde starts - // touching something new, this fails loudly: extend the freeze (and this - // list) together. - const FROZEN_SURFACE = new Set([ - 'Map.prototype.Symbol(Symbol.iterator)', - '%MapIteratorPrototype%.next', - 'Set.prototype.Symbol(Symbol.iterator)', - '%SetIteratorPrototype%.next', - 'Date.prototype.getDate', - 'Date.prototype.toISOString', - '%TypedArray%.prototype.buffer', - '%TypedArray%.prototype.byteOffset', - '%TypedArray%.prototype.byteLength', - 'ArrayBuffer.prototype.byteLength', - ]); - - it('for Map, Set, Date, typed arrays, and ArrayBuffer', async () => { - // Unfrozen realm: the recorders themselves need to redefine members. - const { context, workflowGlobal } = makeContext({ freeze: false }); - const g = workflowGlobal as any; - const touched = new Set(); - - const wrapPrototype = (prototype: object, label: string) => { - for (const key of Reflect.ownKeys(prototype)) { - if (key === 'constructor') continue; - const descriptor = Object.getOwnPropertyDescriptor(prototype, key); - if (!descriptor || !descriptor.configurable) continue; - const name = `${label}.${String(key)}`; - if (typeof descriptor.value === 'function') { - const original = descriptor.value; - Object.defineProperty(prototype, key, { - ...descriptor, - value: function (this: unknown, ...args: unknown[]) { - touched.add(name); - return original.apply(this, args); - }, - }); - } else if (descriptor.get) { - const originalGet = descriptor.get; - Object.defineProperty(prototype, key, { - ...descriptor, - get() { - touched.add(name); - return originalGet.call(this); - }, - set: descriptor.set, - }); - } - } - }; - - const values = vm.runInContext( - `({ - map: new Map([["k", 1]]), - set: new Set([1, 2]), - date: new Date(1234), - f32: new Float32Array([1.5, 2.5]), - u8: new Uint8Array([1, 2, 3]), - ab: new ArrayBuffer(8), - })`, - context - ); - - wrapPrototype(g.Map.prototype, 'Map.prototype'); - wrapPrototype( - Object.getPrototypeOf(new g.Map()[Symbol.iterator]()), - '%MapIteratorPrototype%' - ); - wrapPrototype(g.Set.prototype, 'Set.prototype'); - wrapPrototype( - Object.getPrototypeOf(new g.Set()[Symbol.iterator]()), - '%SetIteratorPrototype%' - ); - const datePrototype = Object.getOwnPropertyDescriptor(g.Date, 'prototype')! - .value as object; - wrapPrototype(datePrototype, 'Date.prototype'); - wrapPrototype( - Object.getPrototypeOf(g.Uint8Array.prototype), - '%TypedArray%.prototype' - ); - wrapPrototype(g.Uint8Array.prototype, 'Uint8Array.prototype'); - wrapPrototype(g.Float32Array.prototype, 'Float32Array.prototype'); - wrapPrototype(g.ArrayBuffer.prototype, 'ArrayBuffer.prototype'); - - for (const value of Object.values(values as Record)) { - touched.clear(); - await dehydrateStepArguments( - { args: [value], closureVars: undefined, thisVal: undefined }, - 'wrun_pin_coverage', - undefined, - g, - false, - false - ); - for (const name of touched) { - expect( - FROZEN_SURFACE, - `member outside the frozen surface executed: ${name}` - ).toContain(name); - } - } - }); -}); - -describe('checker execution surface', () => { - it('never invokes live host collection methods', () => { - const { context, workflowGlobal } = makeContext(); - const map = vm.runInContext('new Map([["k", 1]])', context); - - let invoked = 0; - const original = Map.prototype.forEach; - // Simulate workflow code having replaced the reachable host method. - Map.prototype.forEach = function ( - this: Map, - ...args: [any] - ) { - invoked++; - return original.apply(this, args); - }; - try { - // The checker uses the module-captured primordial: same verdict, - // replaced method never runs. - expect(isRetainedSerializationPassive(map, workflowGlobal)).toBe(true); - expect(invoked).toBe(0); - } finally { - Map.prototype.forEach = original; - } - }); - - it('declines typed arrays re-prototyped onto a frozen hostile prototype', () => { - const { context, workflowGlobal } = makeContext(); - const value = vm.runInContext( - `(() => { - globalThis.__retainedTestCalls = 0; - const realGetter = Object.getOwnPropertyDescriptor( - Object.getPrototypeOf(Uint8Array.prototype), "buffer").get; - const hostile = Object.create( - Object.getPrototypeOf(Uint8Array.prototype), - { buffer: { get() { globalThis.__retainedTestCalls++; return realGetter.call(this); } } } - ); - Object.freeze(hostile); - const ta = new Uint8Array([1, 2]); - Object.setPrototypeOf(ta, hostile); - return ta; - })()`, - context - ); - - expect(isRetainedSerializationPassive(value, workflowGlobal)).toBe(false); - expect(vm.runInContext('globalThis.__retainedTestCalls', context)).toBe(0); - }); -}); - -describe('bigint serialization', () => { - it('declines retention while host BigInt.prototype.toString is replaced', () => { - const { context, workflowGlobal } = makeContext(); - const value = vm.runInContext('({ big: 42n })', context); - expect(isRetainedSerializationPassive(value, workflowGlobal)).toBe(true); - - const original = BigInt.prototype.toString; - // biome-ignore lint/suspicious/noGlobalAssign: simulating workflow-realm tampering - BigInt.prototype.toString = function (this: bigint, ...args: [number?]) { - return original.apply(this, args); - }; - try { - expect(isRetainedSerializationPassive(value, workflowGlobal)).toBe(false); - } finally { - BigInt.prototype.toString = original; - } - - expect(isRetainedSerializationPassive(value, workflowGlobal)).toBe(true); - }); -}); - -describe('walker primordials', () => { - it('uses captured primordials, not live host statics', () => { - const { context, workflowGlobal } = makeContext(); - const map = vm.runInContext('new Map([["k", { ok: true }]])', context); - - // If the walker consulted the live host statics, these would throw. - const originalDescriptor = Object.getOwnPropertyDescriptor; - const originalOwnKeys = Reflect.ownKeys; - (Object as any).getOwnPropertyDescriptor = () => { - throw new Error('live getOwnPropertyDescriptor used'); - }; - (Reflect as any).ownKeys = () => { - throw new Error('live ownKeys used'); - }; - try { - expect(isRetainedSerializationPassive(map, workflowGlobal)).toBe(true); - } finally { - (Object as any).getOwnPropertyDescriptor = originalDescriptor; - (Reflect as any).ownKeys = originalOwnKeys; - } - }); - - it('declines when host Function.prototype carries serializer statics', () => { - const { context, workflowGlobal } = makeContext(); - const plain = vm.runInContext('({ ok: true })', context); - expect(isRetainedSerializationPassive(plain, workflowGlobal)).toBe(true); - - Object.defineProperty( - Function.prototype, - Symbol.for('workflow-serialize'), - { - get() { - return undefined; - }, - configurable: true, - } - ); - try { - expect(isRetainedSerializationPassive(plain, workflowGlobal)).toBe(false); - } finally { - delete (Function.prototype as any)[Symbol.for('workflow-serialize')]; - } - - expect(isRetainedSerializationPassive(plain, workflowGlobal)).toBe(true); - }); -}); diff --git a/packages/core/src/runtime/retained-step-input.ts b/packages/core/src/runtime/retained-step-input.ts deleted file mode 100644 index 7fec0156da..0000000000 --- a/packages/core/src/runtime/retained-step-input.ts +++ /dev/null @@ -1,427 +0,0 @@ -import { types } from 'node:util'; -import { WORKFLOW_SERIALIZE } from '@workflow/serde'; - -// Host constructors that serialization dispatches on (`value instanceof -// global.X`), plus Object/Array, whose statics and prototypes the class -// reducer and devalue's tag lookup read for host-prototype values (hydrated -// step results are host-realm plain objects). The workflow VM's own -// intrinsics are frozen (see vm/index.ts), but host intrinsics are shared -// with the whole process and cannot be frozen — and workflow code can reach -// them (exposed host classes, `structuredClone` results) and plant -// workflow-realm hooks. Verified instead: any dirt declines retention, so a -// planted hook can never execute while a retained VM's inputs serialize. -const HOST_DISPATCH_CONSTRUCTORS = [ - 'Object', - 'Array', - 'Function', - 'Map', - 'Set', - 'Date', - 'RegExp', - 'ArrayBuffer', - 'SharedArrayBuffer', - 'DataView', - 'Int8Array', - 'Uint8Array', - 'Uint8ClampedArray', - 'Int16Array', - 'Uint16Array', - 'Int32Array', - 'Uint32Array', - 'Float16Array', - 'Float32Array', - 'Float64Array', - 'BigInt64Array', - 'BigUint64Array', - 'Headers', - 'URL', - 'URLSearchParams', - 'DOMException', - 'AbortController', - 'AbortSignal', - 'Request', - 'Response', - 'ReadableStream', - 'WritableStream', - 'TransformStream', -] as const; - -// Host primordials captured at module load — before any workflow code can -// exist in the process — so the checker itself never invokes a live host -// member workflow code could have replaced (host constructors are reachable -// via e.g. `structuredClone(new Map()).constructor` or exposed classes). -const hostGetOwnPropertyDescriptor = Object.getOwnPropertyDescriptor; -const hostGetPrototypeOf = Object.getPrototypeOf; -const hostIsFrozen = Object.isFrozen; -const hostOwnKeys = Reflect.ownKeys; -const hostIsArray = Array.isArray; -const hostIsInteger = Number.isInteger; -const hostNumber = Number; -const hostString = String; -const hostBigIntToString = BigInt.prototype.toString; -const hostMapForEach = Map.prototype.forEach; -const hostSetForEach = Set.prototype.forEach; -// biome-ignore lint/style/noNonNullAssertion: the %TypedArray% buffer getter always exists -const hostTypedArrayBuffer = Object.getOwnPropertyDescriptor( - Object.getPrototypeOf(Uint8Array.prototype), - 'buffer' -)!.get!; - -const TYPED_ARRAY_CONSTRUCTORS = [ - 'Int8Array', - 'Uint8Array', - 'Uint8ClampedArray', - 'Int16Array', - 'Uint16Array', - 'Int32Array', - 'Uint32Array', - 'Float16Array', - 'Float32Array', - 'Float64Array', - 'BigInt64Array', - 'BigUint64Array', -] as const; - -function hasOwn(target: object, key: string | symbol): boolean { - return hostGetOwnPropertyDescriptor(target, key) !== undefined; -} - -function isHostDispatchPristine(): boolean { - const hostGlobal = globalThis as unknown as Record; - for (const name of HOST_DISPATCH_CONSTRUCTORS) { - const constructor = hostGlobal[name]; - if (constructor === undefined) continue; - if (hasOwn(constructor as object, Symbol.hasInstance)) return false; - } - // The class reducer reads `cls[WORKFLOW_SERIALIZE]` / `cls.classId` as - // inherited Gets, so the constructors' whole prototype chains — host - // Function.prototype and Object.prototype — must be clean too. - for (const target of [Object, Array, Function.prototype, Object.prototype]) { - if (hasOwn(target, WORKFLOW_SERIALIZE) || hasOwn(target, 'classId')) { - return false; - } - } - for (const [prototype, constructor] of [ - [Object.prototype, Object], - [Array.prototype, Array], - ] as const) { - if ( - hasOwn(prototype, Symbol.toStringTag) || - ownDataProperty(prototype, 'constructor') !== constructor - ) { - return false; - } - } - // Constructor prototype chains end at host Object.prototype, where an - // added @@hasInstance would be found by dispatch lookup. (Host - // Function.prototype's @@hasInstance is spec non-configurable.) - if (hasOwn(Object.prototype, Symbol.hasInstance)) return false; - // The BigInt reducer calls `.toString()` on bigint primitives from host - // code, which resolves on the HOST BigInt.prototype (primitives are - // realm-less; method lookup uses the running code's realm). - if (ownDataProperty(BigInt.prototype, 'toString') !== hostBigIntToString) { - return false; - } - return true; -} - -// Own data-property read that never performs a property Get — workflow code -// can redefine its globals (or their `prototype` slots) with accessors, and -// validation must not execute workflow-owned code. -function ownDataProperty(target: object, key: string): unknown { - const descriptor = hostGetOwnPropertyDescriptor(target, key); - return descriptor && 'value' in descriptor ? descriptor.value : undefined; -} - -function constructorPrototype( - realmGlobal: Record, - constructorName: string -): object | undefined { - const constructor = ownDataProperty(realmGlobal, constructorName); - if ( - (typeof constructor !== 'function' && typeof constructor !== 'object') || - constructor === null || - types.isProxy(constructor) - ) { - return undefined; - } - const prototype = ownDataProperty(constructor, 'prototype'); - return typeof prototype === 'object' && - prototype !== null && - !types.isProxy(prototype) - ? prototype - : undefined; -} - -// The workflow realm's prototype for a supported built-in, but only once the -// sandbox froze it: serialization both executes members on these prototypes -// (iterators, getters) and reads others (`constructor`), so a mutable -// prototype — including one in a realm where freezeSerializationIntrinsics -// never ran — is not provably passive. Host-realm instances of these -// built-ins decline for the same reason: host prototypes cannot be frozen -// and are reachable from workflow code (e.g. via structuredClone results). -function frozenRealmPrototype( - workflowGlobal: Record, - constructorName: string -): object | undefined { - const prototype = constructorPrototype(workflowGlobal, constructorName); - return prototype !== undefined && hostIsFrozen(prototype) - ? prototype - : undefined; -} - -function hasAllowedPrototype( - value: object, - workflowGlobal: Record, - constructorName: string -): boolean { - const prototype = hostGetPrototypeOf(value); - return ( - prototype === - constructorPrototype( - globalThis as unknown as Record, - constructorName - ) || prototype === constructorPrototype(workflowGlobal, constructorName) - ); -} - -function isArrayIndex(key: string): boolean { - const index = hostNumber(key); - return ( - hostIsInteger(index) && - index >= 0 && - index < 2 ** 32 - 1 && - hostString(index) === key - ); -} - -function isPassiveArrayProperty( - array: unknown[], - key: string | symbol, - workflowGlobal: Record, - seen: WeakSet -): boolean { - if (key === 'length') return true; - const descriptor = hostGetOwnPropertyDescriptor(array, key); - if (!descriptor) return false; - // Only own enumerable data indices are passive. Anything hidden — symbol - // tags, non-enumerable properties, accessors — can be observed by - // serialization dispatch (reducer probes, thenable checks, the class - // reducer) and can execute workflow code when read. - return ( - typeof key === 'string' && - isArrayIndex(key) && - descriptor.enumerable === true && - 'value' in descriptor && - isPassive(descriptor.value, workflowGlobal, seen) - ); -} - -function isPassiveArray( - value: unknown[], - workflowGlobal: Record, - seen: WeakSet -): boolean { - return ( - hasAllowedPrototype(value, workflowGlobal, 'Array') && - hostOwnKeys(value).every((key) => - isPassiveArrayProperty(value, key, workflowGlobal, seen) - ) - ); -} - -// Instances of the supported built-ins must carry no own properties at all: -// serialization never reads own properties on them, but an own accessor or -// symbol could still be observed through other dispatch lookups, and clean -// instances are the overwhelmingly common case anyway. -function hasNoOwnProperties(value: object): boolean { - return hostOwnKeys(value).length === 0; -} - -function isPassiveMap( - value: Map, - workflowGlobal: Record, - seen: WeakSet -): boolean { - const prototype = hostGetPrototypeOf(value); - if (prototype !== frozenRealmPrototype(workflowGlobal, 'Map')) return false; - if (!hasNoOwnProperties(value)) return false; - let passive = true; - // The captured host forEach iterates via internal slots — no realm - // members (and no live, replaceable host members) execute. - hostMapForEach.call(value, (entryValue: unknown, key: unknown) => { - passive &&= - isPassive(key, workflowGlobal, seen) && - isPassive(entryValue, workflowGlobal, seen); - }); - return passive; -} - -function isPassiveSet( - value: Set, - workflowGlobal: Record, - seen: WeakSet -): boolean { - const prototype = hostGetPrototypeOf(value); - if (prototype !== frozenRealmPrototype(workflowGlobal, 'Set')) return false; - if (!hasNoOwnProperties(value)) return false; - let passive = true; - hostSetForEach.call(value, (entryValue: unknown) => { - passive &&= isPassive(entryValue, workflowGlobal, seen); - }); - return passive; -} - -function isPassiveDate( - value: object, - workflowGlobal: Record -): boolean { - const prototype = hostGetPrototypeOf(value); - return ( - prototype === frozenRealmPrototype(workflowGlobal, 'Date') && - hasNoOwnProperties(value) - ); -} - -function isPassiveTypedArray( - value: object, - workflowGlobal: Record -): boolean { - // Identity against the finite set of the realm's real (frozen) typed-array - // prototypes — NOT "anything chaining to %TypedArray%": a workflow can - // manufacture a frozen hostile prototype with a delegating `buffer` getter - // and setPrototypeOf a real typed array onto it. - const prototype = hostGetPrototypeOf(value); - if ( - !TYPED_ARRAY_CONSTRUCTORS.some( - (name) => prototype === frozenRealmPrototype(workflowGlobal, name) - ) - ) { - return false; - } - // Own keys on a typed array are exactly its canonical indices. - if ( - !hostOwnKeys(value).every( - (key) => typeof key === 'string' && isArrayIndex(key) - ) - ) { - return false; - } - // Read the backing buffer via the captured host getter (internal slots - // work cross-realm); reject SharedArrayBuffer backing. - return !types.isSharedArrayBuffer(hostTypedArrayBuffer.call(value)); -} - -function isPassiveArrayBuffer( - value: object, - workflowGlobal: Record -): boolean { - const prototype = hostGetPrototypeOf(value); - return ( - prototype === frozenRealmPrototype(workflowGlobal, 'ArrayBuffer') && - hasNoOwnProperties(value) - ); -} - -function isPassiveObjectProperty( - object: object, - key: string | symbol, - workflowGlobal: Record, - seen: WeakSet -): boolean { - const descriptor = hostGetOwnPropertyDescriptor(object, key); - if (!descriptor) return false; - // Only own enumerable string-keyed data properties are passive. Anything - // hidden — symbol tags, non-enumerable properties, accessors — can be - // observed by serialization dispatch (reducer probes like `.signal`, - // thenable checks, the class reducer) and can execute workflow code. - return ( - typeof key === 'string' && - key !== '__proto__' && - descriptor.enumerable === true && - 'value' in descriptor && - isPassive(descriptor.value, workflowGlobal, seen) - ); -} - -function isPassivePlainObject( - value: object, - workflowGlobal: Record, - seen: WeakSet -): boolean { - if (!hasAllowedPrototype(value, workflowGlobal, 'Object')) return false; - return hostOwnKeys(value).every((key) => - isPassiveObjectProperty(value, key, workflowGlobal, seen) - ); -} - -/** - * Whether serializing `value` through the ordinary pipeline provably executes - * no workflow code and draws no workflow-realm randomness. - * - * Retained sessions keep running after suspension, so serialization side - * effects there would desync the live VM from what a cold replay - * reconstructs (replay never re-serializes an already-created step). The - * bytes themselves cannot differ by mode — serialization happens once and - * every mode reads the same `step_created` event — so passivity is the only - * property retention needs. - * - * Passive values: primitives; plain objects and arrays (own enumerable - * string-keyed data properties only — devalue traverses these purely via own - * reads); and workflow-realm Map/Set/Date/typed arrays/ArrayBuffer whose - * prototypes the sandbox froze (see freezeSerializationIntrinsics). - * Everything else — proxies, accessors, functions, custom classes, RegExp, - * hidden keys — declines, serializes exactly the same way, and the session - * falls back to ordinary replay for that boundary. - */ -function isPassive( - value: unknown, - workflowGlobal: Record, - seen: WeakSet -): boolean { - if ( - value === null || - value === undefined || - typeof value === 'boolean' || - typeof value === 'bigint' || - typeof value === 'number' || - typeof value === 'string' - ) { - return true; - } - if (typeof value !== 'object' || types.isProxy(value)) return false; - if (seen.has(value)) return true; - seen.add(value); - - if (hostIsArray(value)) return isPassiveArray(value, workflowGlobal, seen); - if (types.isMap(value)) return isPassiveMap(value, workflowGlobal, seen); - if (types.isSet(value)) return isPassiveSet(value, workflowGlobal, seen); - if (types.isDate(value)) return isPassiveDate(value, workflowGlobal); - if (types.isTypedArray(value)) { - return isPassiveTypedArray(value, workflowGlobal); - } - if (types.isArrayBuffer(value)) { - return isPassiveArrayBuffer(value, workflowGlobal); - } - if ( - types.isSharedArrayBuffer(value) || - types.isRegExp(value) || - types.isDataView(value) || - types.isBoxedPrimitive(value) || - types.isNativeError(value) || - types.isPromise(value) || - types.isArgumentsObject(value) - ) { - return false; - } - return isPassivePlainObject(value, workflowGlobal, seen); -} - -export function isRetainedSerializationPassive( - value: unknown, - workflowGlobal: Record -): boolean { - return ( - isHostDispatchPristine() && isPassive(value, workflowGlobal, new WeakSet()) - ); -} diff --git a/packages/core/src/runtime/suspension-handler.ts b/packages/core/src/runtime/suspension-handler.ts index 64c362c566..c23ab2e878 100644 --- a/packages/core/src/runtime/suspension-handler.ts +++ b/packages/core/src/runtime/suspension-handler.ts @@ -31,7 +31,19 @@ import * as Attribute from '../telemetry/semantic-conventions.js'; import { getAbortStreamIdFromToken } from '../util.js'; import { getMaxInlineSteps } from './constants.js'; import { type MutableEventLog, withPreconditionRetry } from './helpers.js'; -import { isRetainedSerializationPassive } from './retained-step-input.js'; + +// Serializing a primitive executes no code of any kind. BigInt is excluded: +// its encoding calls a prototype method. Widened to plain data and standard +// built-ins by the retained-input walker in a follow-up. +function isPrimitiveStepArgument(value: unknown): boolean { + return ( + value === null || + value === undefined || + typeof value === 'boolean' || + typeof value === 'number' || + typeof value === 'string' + ); +} export interface SuspensionHandlerParams { suspension: WorkflowSuspension; @@ -502,22 +514,19 @@ export async function handleSuspension({ // durable bytes cannot depend on retention. What retention needs to know is // whether that serialization will execute workflow code (getters, hooks, // patched prototype members) — side effects a cold replay would not repeat. - // If any input in the batch is not provably passive, the caller demotes the - // session so the side effects land in a VM that is about to be discarded, - // exactly like the pre-retention runtime. + // For now only primitive arguments are provably passive (serializing them + // executes no code at all); a follow-up widens this to plain data and the + // standard built-ins. If any input in the batch is not provably passive, + // the caller demotes the session so the side effects land in a VM that is + // about to be discarded, exactly like the pre-retention runtime. let retainedStepInputsSafe = true; if (prepareForRetention) { for (const queueItem of stepItems) { if (!stepsNeedingCreation.has(queueItem.correlationId)) continue; if ( - !isRetainedSerializationPassive( - { - args: queueItem.args, - closureVars: queueItem.closureVars, - thisVal: queueItem.thisVal, - }, - suspension.globalThis - ) + queueItem.thisVal !== undefined || + queueItem.closureVars !== undefined || + !queueItem.args.every(isPrimitiveStepArgument) ) { retainedStepInputsSafe = false; break; diff --git a/packages/core/src/workflow.test.ts b/packages/core/src/workflow.test.ts index 688bc2774d..6830603819 100644 --- a/packages/core/src/workflow.test.ts +++ b/packages/core/src/workflow.test.ts @@ -405,6 +405,68 @@ describe('runWorkflow', () => { ).toEqual({ type: 'replay' }); }); + it('falls back to replay (not failure) after mid-execution divergence', async () => { + const ops: Promise[] = []; + const workflowRun: WorkflowRun = { + runId: 'wrun_retained_mid_divergence', + workflowName: 'workflow', + status: 'running', + input: await dehydrateWorkflowArguments( + [], + 'wrun_retained_mid_divergence', + noEncryptionKey, + ops + ), + createdAt: new Date('2024-01-01T00:00:00.000Z'), + updatedAt: new Date('2024-01-01T00:00:00.000Z'), + startedAt: new Date('2024-01-01T00:00:00.000Z'), + deploymentId: 'test-deployment', + }; + const workflowCode = ` + const step = globalThis[Symbol.for("WORKFLOW_USE_STEP")]("step"); + async function workflow() { await step(); } + ${getWorkflowTransformCode('workflow')}`; + const events: Event[] = []; + + const suspended = await executeWorkflow({ + type: 'replay', + workflowCode, + workflowRun, + events, + encryptionKey: noEncryptionKey, + replayPayloadCache: new ReplayPayloadCache(noEncryptionKey), + }); + assert(suspended.type === 'suspended'); + + // A strict extension whose appended suffix the VM cannot consume: the + // resume itself starts, then diverges mid-execution. + await expect( + executeWorkflow({ + type: 'resume', + session: suspended.session, + events: [ + { + eventId: 'event-alien', + runId: workflowRun.runId, + eventType: 'hook_received', + correlationId: 'hook_unknown', + eventData: {}, + createdAt: new Date('2024-01-01T00:00:01.000Z'), + } as Event, + ], + }) + ).rejects.toThrow(/could not consume event/); + + // The session must demote to replay — not throw "cannot resume". + expect( + await executeWorkflow({ + type: 'resume', + session: suspended.session, + events: [], + }) + ).toEqual({ type: 'replay' }); + }); + it('regenerates step correlation IDs independent of startedAt (turbo replay-stability)', async () => { // Turbo's first delivery synthesizes `startedAt` from the local clock, // while later (non-turbo) deliveries load the server-canonical `startedAt`. diff --git a/packages/core/src/workflow.ts b/packages/core/src/workflow.ts index 539b2e199a..57f75435fe 100644 --- a/packages/core/src/workflow.ts +++ b/packages/core/src/workflow.ts @@ -151,31 +151,6 @@ type WorkflowSessionState = | { readonly type: 'replay' } | { readonly type: 'completed' }; -// The suspension counts are all derived from `steps` in the WorkflowSuspension -// constructor, so identical items mean an identical boundary. -function isSameSuspensionBoundary( - previous: WorkflowSuspension, - next: WorkflowSuspension -): boolean { - return ( - previous.steps.length === next.steps.length && - previous.steps.every((item, index) => item === next.steps[index]) - ); -} - -// A suspended VM is quiescent, but each parked step consumer signals its own -// (identical) suspension via scheduleWhenIdle — absorb those duplicates and -// treat anything else as unretainable. -function updateSuspendedSession( - suspension: WorkflowSuspension, - error: Error -): WorkflowSessionState { - return WorkflowSuspension.is(error) && - isSameSuspensionBoundary(suspension, error) - ? { type: 'suspended', suspension } - : { type: 'replay' }; -} - export interface WorkflowSession { readonly workflowRun: WorkflowRun; readonly argumentCount: number; @@ -322,7 +297,7 @@ export async function runWorkflow( return result.output; } -function createWorkflowSession({ +async function createWorkflowSession({ workflowCode, workflowRun, events, @@ -334,311 +309,303 @@ function createWorkflowSession({ session: WorkflowSession; execution: Promise; }> { - return (async () => { - const startedAt = workflowRun.startedAt; - if (!startedAt) { - throw new WorkflowRuntimeError( - `Workflow run "${workflowRun.runId}" has no "startedAt" timestamp (should not happen)` - ); - } - - // The deterministic RNG seed is derived from identifiers that are all - // known the instant the queue message arrives — `runId`, `workflowName`, - // and `deploymentId` — with no timestamp component. `runId` alone already - // makes the seed unique-per-run and replay-stable; `workflowName` and - // `deploymentId` are included for extra entropy. Dropping the timestamp - // means the seed no longer depends on `startedAt`/`createdAt`, so it (and - // the VM context) can be computed before any server round-trip. - // - // The VM's initial fixed clock is derived from the run's creation time, - // recovered from the ULID embedded in `runId` (also available immediately), - // falling back to the run snapshot's `createdAt` for non-ULID ids. This - // initial `fixedTimestamp` only governs `Date.now()` / `new Date()` in the - // window before the first event is consumed; thereafter `updateTimestamp` - // advances the VM clock to each consumed event's `createdAt` (see the - // EventsConsumer below), starting with `run_created`. - const fixedTimestamp = - runIdCreatedAt(workflowRun.runId) ?? +workflowRun.createdAt; - - // Get the port before creating VM context to avoid async operations - // affecting the deterministic timestamp - const isVercel = process.env.VERCEL_URL !== undefined; - // Load getPort lazily to prevent Turbopack from tracing get-port's - // fs ops (readdir, readFile) into the flow route bundle. The resolved - // port is cached per process (see get-port-lazy.ts), so this is cheap - // on replays after the first. - const workflowBaseUrl = createWorkflowBaseUrl( - isVercel - ? `https://${process.env.VERCEL_URL}` - : `http://localhost:${(await getPortLazy()) ?? 3000}` + const startedAt = workflowRun.startedAt; + if (!startedAt) { + throw new WorkflowRuntimeError( + `Workflow run "${workflowRun.runId}" has no "startedAt" timestamp (should not happen)` ); + } - const { - context, - globalThis: vmGlobalThis, - updateTimestamp, - } = createContext({ - seed: `${workflowRun.runId}:${workflowRun.workflowName}:${workflowRun.deploymentId}`, - fixedTimestamp, - }); + // The deterministic RNG seed is derived from identifiers that are all + // known the instant the queue message arrives — `runId`, `workflowName`, + // and `deploymentId` — with no timestamp component. `runId` alone already + // makes the seed unique-per-run and replay-stable; `workflowName` and + // `deploymentId` are included for extra entropy. Dropping the timestamp + // means the seed no longer depends on `startedAt`/`createdAt`, so it (and + // the VM context) can be computed before any server round-trip. + // + // The VM's initial fixed clock is derived from the run's creation time, + // recovered from the ULID embedded in `runId` (also available immediately), + // falling back to the run snapshot's `createdAt` for non-ULID ids. This + // initial `fixedTimestamp` only governs `Date.now()` / `new Date()` in the + // window before the first event is consumed; thereafter `updateTimestamp` + // advances the VM clock to each consumed event's `createdAt` (see the + // EventsConsumer below), starting with `run_created`. + const fixedTimestamp = + runIdCreatedAt(workflowRun.runId) ?? +workflowRun.createdAt; + + // Get the port before creating VM context to avoid async operations + // affecting the deterministic timestamp + const isVercel = process.env.VERCEL_URL !== undefined; + // Load getPort lazily to prevent Turbopack from tracing get-port's + // fs ops (readdir, readFile) into the flow route bundle. The resolved + // port is cached per process (see get-port-lazy.ts), so this is cheap + // on replays after the first. + const workflowBaseUrl = createWorkflowBaseUrl( + isVercel + ? `https://${process.env.VERCEL_URL}` + : `http://localhost:${(await getPortLazy()) ?? 3000}` + ); - const initialInterruption = withResolvers(); - let state: WorkflowSessionState = { - type: 'running', - interruption: initialInterruption, - }; + const { + context, + globalThis: vmGlobalThis, + updateTimestamp, + } = createContext({ + seed: `${workflowRun.runId}:${workflowRun.workflowName}:${workflowRun.deploymentId}`, + fixedTimestamp, + }); - const onWorkflowError = (error: Error): void => { - switch (state.type) { - case 'running': { - const { interruption } = state; - state = WorkflowSuspension.is(error) - ? { type: 'suspended', suspension: error } - : { type: 'replay' }; - interruption.reject(error); - return; - } - case 'suspended': - state = updateSuspendedSession(state.suspension, error); - return; - case 'replay': - case 'completed': - return; + const initialInterruption = withResolvers(); + let state: WorkflowSessionState = { + type: 'running', + interruption: initialInterruption, + }; + + const onWorkflowError = (error: Error): void => { + switch (state.type) { + case 'running': { + const { interruption } = state; + state = WorkflowSuspension.is(error) + ? { type: 'suspended', suspension: error } + : { type: 'replay' }; + interruption.reject(error); + return; } - state satisfies never; - }; - - const ulid = monotonicFactory(() => vmGlobalThis.Math.random()); - const generateNanoid = nanoid.customRandom(nanoid.urlAlphabet, 21, (size) => - new Uint8Array(size).map(() => 256 * vmGlobalThis.Math.random()) - ); - - // Create a mutable holder for the promise queue so the EventsConsumer - // can access the current queue state via a getter. The queue is mutated - // by step/hook/sleep callbacks as events are processed. - const promiseQueueHolder = { current: Promise.resolve() }; - - const eventsConsumer = new EventsConsumer(events, { - onConsumedEvent: (event) => { - updateTimestamp(+event.createdAt); - }, - onUnconsumedEvent: (event) => { - onWorkflowError( - new ReplayDivergenceError( - `Replay could not consume event: eventType=${event.eventType}, correlationId=${event.correlationId}, eventId=${event.eventId}.`, - { eventId: event.eventId } - ) - ); - }, - getPromiseQueue: () => promiseQueueHolder.current, - }); - - const workflowContext: WorkflowOrchestratorContext = { - runId: workflowRun.runId, - encryptionKey, - worldCapabilities, - globalThis: vmGlobalThis, - onWorkflowError, - eventsConsumer, - // Correlation IDs (step_/wait_/hook_) are derived from `generateUlid`, so - // the time prefix fed to `ulid()` MUST be replay-stable across every - // delivery — otherwise a redelivery regenerates different correlation IDs - // and replay throws ReplayDivergenceError. `startedAt` is NOT safe here: - // under turbo the first delivery synthesizes `startedAt` from the local - // clock, but later (non-turbo) deliveries load the server-canonical - // `startedAt`, which differs by >=1ms. Use the same replay-stable value - // that already seeds the RNG and the VM clock (`fixedTimestamp`, recovered - // from the run ID's ULID and known the instant the message arrives). - generateUlid: () => ulid(fixedTimestamp), - generateNanoid, - invocationsQueue: new Map(), - // Use getter/setter so the EventsConsumer's getPromiseQueue() always - // sees the latest queue state as it's mutated by step/hook/sleep callbacks. - get promiseQueue() { - return promiseQueueHolder.current; - }, - set promiseQueue(value: Promise) { - promiseQueueHolder.current = value; - }, - pendingDeliveries: 0, - suspensionGeneration: 0, - pendingDeliveryBarriers: new Map(), - replayPayloadCache, - }; - - // Consume run lifecycle events - these are structural events that don't - // need special handling in the workflow, but must be consumed to advance - // past them in the event log - workflowContext.eventsConsumer.subscribe((event) => { - if (!event) { - return EventConsumerResult.NotConsumed; + case 'suspended': { + // A suspended VM is quiescent, but each parked step consumer + // signals its own (identical) suspension via scheduleWhenIdle — + // absorb those duplicates, treat anything else as unretainable. + // The suspension counts are all derived from `steps`, so identical + // items mean an identical boundary. + const { steps } = state.suspension; + const isDuplicateSignal = + WorkflowSuspension.is(error) && + error.steps.length === steps.length && + error.steps.every((item, index) => item === steps[index]); + if (!isDuplicateSignal) state = { type: 'replay' }; + return; } + case 'replay': + case 'completed': + return; + } + state satisfies never; + }; - // Consume run_created - every run has exactly one - if (event.eventType === 'run_created') { - return EventConsumerResult.Consumed; - } + const ulid = monotonicFactory(() => vmGlobalThis.Math.random()); + const generateNanoid = nanoid.customRandom(nanoid.urlAlphabet, 21, (size) => + new Uint8Array(size).map(() => 256 * vmGlobalThis.Math.random()) + ); - // Consume run_started - every run has exactly one - if (event.eventType === 'run_started') { - return EventConsumerResult.Consumed; - } + // Create a mutable holder for the promise queue so the EventsConsumer + // can access the current queue state via a getter. The queue is mutated + // by step/hook/sleep callbacks as events are processed. + const promiseQueueHolder = { current: Promise.resolve() }; + + const eventsConsumer = new EventsConsumer(events, { + onConsumedEvent: (event) => { + updateTimestamp(+event.createdAt); + }, + onUnconsumedEvent: (event) => { + onWorkflowError( + new ReplayDivergenceError( + `Replay could not consume event: eventType=${event.eventType}, correlationId=${event.correlationId}, eventId=${event.eventId}.`, + { eventId: event.eventId } + ) + ); + }, + getPromiseQueue: () => promiseQueueHolder.current, + }); - // Attribute writes performed from a step have no workflow-body call to - // consume them during replay; they are already reflected in the run - // snapshot and remain structural until a read API is introduced. - if ( - event.eventType === 'attr_set' && - event.eventData.writer.type === 'step' - ) { - return EventConsumerResult.Consumed; - } + const workflowContext: WorkflowOrchestratorContext = { + runId: workflowRun.runId, + encryptionKey, + worldCapabilities, + globalThis: vmGlobalThis, + onWorkflowError, + eventsConsumer, + // Correlation IDs (step_/wait_/hook_) are derived from `generateUlid`, so + // the time prefix fed to `ulid()` MUST be replay-stable across every + // delivery — otherwise a redelivery regenerates different correlation IDs + // and replay throws ReplayDivergenceError. `startedAt` is NOT safe here: + // under turbo the first delivery synthesizes `startedAt` from the local + // clock, but later (non-turbo) deliveries load the server-canonical + // `startedAt`, which differs by >=1ms. Use the same replay-stable value + // that already seeds the RNG and the VM clock (`fixedTimestamp`, recovered + // from the run ID's ULID and known the instant the message arrives). + generateUlid: () => ulid(fixedTimestamp), + generateNanoid, + invocationsQueue: new Map(), + // Use getter/setter so the EventsConsumer's getPromiseQueue() always + // sees the latest queue state as it's mutated by step/hook/sleep callbacks. + get promiseQueue() { + return promiseQueueHolder.current; + }, + set promiseQueue(value: Promise) { + promiseQueueHolder.current = value; + }, + pendingDeliveries: 0, + suspensionGeneration: 0, + pendingDeliveryBarriers: new Map(), + replayPayloadCache, + }; + // Consume run lifecycle events - these are structural events that don't + // need special handling in the workflow, but must be consumed to advance + // past them in the event log + workflowContext.eventsConsumer.subscribe((event) => { + if (!event) { return EventConsumerResult.NotConsumed; - }); + } - const useStep = createUseStep(workflowContext); - const createHook = createCreateHook(workflowContext); - const sleep = createSleep(workflowContext); - const setAttributes = createSetAttributes(workflowContext); - - // @ts-expect-error - `@types/node` says symbol is not valid, but it does work - vmGlobalThis[WORKFLOW_USE_STEP] = useStep; - // @ts-expect-error - `@types/node` says symbol is not valid, but it does work - vmGlobalThis[WORKFLOW_SET_ATTRIBUTES] = setAttributes; - // @ts-expect-error - `@types/node` says symbol is not valid, but it does work - vmGlobalThis[WORKFLOW_CREATE_HOOK] = createHook; - // @ts-expect-error - `@types/node` says symbol is not valid, but it does work - vmGlobalThis[WORKFLOW_SLEEP] = sleep; - // @ts-expect-error - `@types/node` says symbol is not valid, but it does work - vmGlobalThis[WORKFLOW_GET_STREAM_ID] = (namespace?: string) => - getWorkflowRunStreamId(workflowRun.runId, namespace); - - // For the workflow VM, we store the context in a symbol on the `globalThis` object - const ctx: WorkflowMetadata = { - workflowName: workflowRun.workflowName, - workflowRunId: workflowRun.runId, - workflowStartedAt: new vmGlobalThis.Date(+startedAt), - url: workflowBaseUrl, - features: { encryption: !!encryptionKey }, - }; + // Consume run_created - every run has exactly one + if (event.eventType === 'run_created') { + return EventConsumerResult.Consumed; + } - // @ts-expect-error - `@types/node` says symbol is not valid, but it does work - vmGlobalThis[WORKFLOW_CONTEXT_SYMBOL] = ctx; - // @ts-expect-error - `@types/node` says symbol is not valid, but it does work - vmGlobalThis[STABLE_ULID] = ulid; + // Consume run_started - every run has exactly one + if (event.eventType === 'run_started') { + return EventConsumerResult.Consumed; + } - // NOTE: Will have a config override to use the custom fetch step. - // For now `fetch` must be explicitly imported from `workflow`. - vmGlobalThis.fetch = () => { - throw new vmGlobalThis.Error( - `Global "fetch" is unavailable in workflow functions. Use the "fetch" step function from "workflow" to make HTTP requests.\n\nLearn more: https://workflow-sdk.dev/err/${ERROR_SLUGS.FETCH_IN_WORKFLOW_FUNCTION}` - ); - }; + // Attribute writes performed from a step have no workflow-body call to + // consume them during replay; they are already reflected in the run + // snapshot and remain structural until a read API is introduced. + if ( + event.eventType === 'attr_set' && + event.eventData.writer.type === 'step' + ) { + return EventConsumerResult.Consumed; + } - // Override timeout/interval functions to throw helpful errors - // These are not supported in workflow functions because they rely on - // asynchronous scheduling which breaks deterministic replay - const timeoutErrorMessage = - 'Timeout functions like "setTimeout" and "setInterval" are not supported in workflow functions. Use the "sleep" function from "workflow" for time-based delays.'; + return EventConsumerResult.NotConsumed; + }); - (vmGlobalThis as any).setTimeout = () => { - throw new WorkflowRuntimeError(timeoutErrorMessage, { - slug: ERROR_SLUGS.TIMEOUT_FUNCTIONS_IN_WORKFLOW, - }); - }; - (vmGlobalThis as any).setInterval = () => { - throw new WorkflowRuntimeError(timeoutErrorMessage, { - slug: ERROR_SLUGS.TIMEOUT_FUNCTIONS_IN_WORKFLOW, - }); - }; - (vmGlobalThis as any).clearTimeout = () => { - throw new WorkflowRuntimeError(timeoutErrorMessage, { - slug: ERROR_SLUGS.TIMEOUT_FUNCTIONS_IN_WORKFLOW, - }); - }; - (vmGlobalThis as any).clearInterval = () => { - throw new WorkflowRuntimeError(timeoutErrorMessage, { - slug: ERROR_SLUGS.TIMEOUT_FUNCTIONS_IN_WORKFLOW, - }); - }; - (vmGlobalThis as any).setImmediate = () => { - throw new WorkflowRuntimeError(timeoutErrorMessage, { - slug: ERROR_SLUGS.TIMEOUT_FUNCTIONS_IN_WORKFLOW, - }); - }; - (vmGlobalThis as any).clearImmediate = () => { - throw new WorkflowRuntimeError(timeoutErrorMessage, { - slug: ERROR_SLUGS.TIMEOUT_FUNCTIONS_IN_WORKFLOW, - }); - }; + const useStep = createUseStep(workflowContext); + const createHook = createCreateHook(workflowContext); + const sleep = createSleep(workflowContext); + const setAttributes = createSetAttributes(workflowContext); + + // @ts-expect-error - `@types/node` says symbol is not valid, but it does work + vmGlobalThis[WORKFLOW_USE_STEP] = useStep; + // @ts-expect-error - `@types/node` says symbol is not valid, but it does work + vmGlobalThis[WORKFLOW_SET_ATTRIBUTES] = setAttributes; + // @ts-expect-error - `@types/node` says symbol is not valid, but it does work + vmGlobalThis[WORKFLOW_CREATE_HOOK] = createHook; + // @ts-expect-error - `@types/node` says symbol is not valid, but it does work + vmGlobalThis[WORKFLOW_SLEEP] = sleep; + // @ts-expect-error - `@types/node` says symbol is not valid, but it does work + vmGlobalThis[WORKFLOW_GET_STREAM_ID] = (namespace?: string) => + getWorkflowRunStreamId(workflowRun.runId, namespace); + + // For the workflow VM, we store the context in a symbol on the `globalThis` object + const ctx: WorkflowMetadata = { + workflowName: workflowRun.workflowName, + workflowRunId: workflowRun.runId, + workflowStartedAt: new vmGlobalThis.Date(+startedAt), + url: workflowBaseUrl, + features: { encryption: !!encryptionKey }, + }; + + // @ts-expect-error - `@types/node` says symbol is not valid, but it does work + vmGlobalThis[WORKFLOW_CONTEXT_SYMBOL] = ctx; + // @ts-expect-error - `@types/node` says symbol is not valid, but it does work + vmGlobalThis[STABLE_ULID] = ulid; + + // NOTE: Will have a config override to use the custom fetch step. + // For now `fetch` must be explicitly imported from `workflow`. + vmGlobalThis.fetch = () => { + throw new vmGlobalThis.Error( + `Global "fetch" is unavailable in workflow functions. Use the "fetch" step function from "workflow" to make HTTP requests.\n\nLearn more: https://workflow-sdk.dev/err/${ERROR_SLUGS.FETCH_IN_WORKFLOW_FUNCTION}` + ); + }; - // `AbortController` and `AbortSignal` in the workflow VM are hook-backed - // for deterministic replay. The controller's abort() queues a hook resumption, - // and signal.aborted is updated when the hook event is processed during replay. - (vmGlobalThis as any).AbortController = - createCreateAbortController(workflowContext); - const abortSignalStatics = createAbortSignalStatics(); - (vmGlobalThis as any).AbortSignal = { - abort: abortSignalStatics.abort, - any: abortSignalStatics.any, - timeout: abortSignalStatics.timeout, - }; + // Override timeout/interval functions to throw helpful errors + // These are not supported in workflow functions because they rely on + // asynchronous scheduling which breaks deterministic replay + const timeoutErrorMessage = + 'Timeout functions like "setTimeout" and "setInterval" are not supported in workflow functions. Use the "sleep" function from "workflow" for time-based delays.'; - // `Request` and `Response` are special built-in classes that invoke steps - // for the `json()`, `text()` and `arrayBuffer()` instance methods - class Request implements globalThis.Request { - cache!: globalThis.Request['cache']; - credentials!: globalThis.Request['credentials']; - destination!: globalThis.Request['destination']; - headers!: Headers; - integrity!: string; - method!: string; - mode!: globalThis.Request['mode']; - redirect!: globalThis.Request['redirect']; - referrer!: string; - referrerPolicy!: globalThis.Request['referrerPolicy']; - url!: string; - keepalive!: boolean; - signal!: AbortSignal; - duplex!: 'half'; - body!: ReadableStream | null; - - constructor(input: any, init?: RequestInit) { - // Handle URL input - if (typeof input === 'string' || input instanceof vmGlobalThis.URL) { - const urlString = String(input); - // Validate URL format - try { - new vmGlobalThis.URL(urlString); - this.url = urlString; - } catch (cause) { - throw new TypeError(`Failed to parse URL from ${urlString}`, { - cause, - }); - } - } else { - // Input is a Request object - clone its properties - this.url = input.url; - if (!init) { - this.method = input.method; - this.headers = new vmGlobalThis.Headers(input.headers); - this.body = input.body; - this.mode = input.mode; - this.credentials = input.credentials; - this.cache = input.cache; - this.redirect = input.redirect; - this.referrer = input.referrer; - this.referrerPolicy = input.referrerPolicy; - this.integrity = input.integrity; - this.keepalive = input.keepalive; - this.signal = input.signal; - this.duplex = input.duplex; - this.destination = input.destination; - return; - } - // If init is provided, merge: use source properties, then override with init - // Copy all properties from the source Request first + (vmGlobalThis as any).setTimeout = () => { + throw new WorkflowRuntimeError(timeoutErrorMessage, { + slug: ERROR_SLUGS.TIMEOUT_FUNCTIONS_IN_WORKFLOW, + }); + }; + (vmGlobalThis as any).setInterval = () => { + throw new WorkflowRuntimeError(timeoutErrorMessage, { + slug: ERROR_SLUGS.TIMEOUT_FUNCTIONS_IN_WORKFLOW, + }); + }; + (vmGlobalThis as any).clearTimeout = () => { + throw new WorkflowRuntimeError(timeoutErrorMessage, { + slug: ERROR_SLUGS.TIMEOUT_FUNCTIONS_IN_WORKFLOW, + }); + }; + (vmGlobalThis as any).clearInterval = () => { + throw new WorkflowRuntimeError(timeoutErrorMessage, { + slug: ERROR_SLUGS.TIMEOUT_FUNCTIONS_IN_WORKFLOW, + }); + }; + (vmGlobalThis as any).setImmediate = () => { + throw new WorkflowRuntimeError(timeoutErrorMessage, { + slug: ERROR_SLUGS.TIMEOUT_FUNCTIONS_IN_WORKFLOW, + }); + }; + (vmGlobalThis as any).clearImmediate = () => { + throw new WorkflowRuntimeError(timeoutErrorMessage, { + slug: ERROR_SLUGS.TIMEOUT_FUNCTIONS_IN_WORKFLOW, + }); + }; + + // `AbortController` and `AbortSignal` in the workflow VM are hook-backed + // for deterministic replay. The controller's abort() queues a hook resumption, + // and signal.aborted is updated when the hook event is processed during replay. + (vmGlobalThis as any).AbortController = + createCreateAbortController(workflowContext); + const abortSignalStatics = createAbortSignalStatics(); + (vmGlobalThis as any).AbortSignal = { + abort: abortSignalStatics.abort, + any: abortSignalStatics.any, + timeout: abortSignalStatics.timeout, + }; + + // `Request` and `Response` are special built-in classes that invoke steps + // for the `json()`, `text()` and `arrayBuffer()` instance methods + class Request implements globalThis.Request { + cache!: globalThis.Request['cache']; + credentials!: globalThis.Request['credentials']; + destination!: globalThis.Request['destination']; + headers!: Headers; + integrity!: string; + method!: string; + mode!: globalThis.Request['mode']; + redirect!: globalThis.Request['redirect']; + referrer!: string; + referrerPolicy!: globalThis.Request['referrerPolicy']; + url!: string; + keepalive!: boolean; + signal!: AbortSignal; + duplex!: 'half'; + body!: ReadableStream | null; + + constructor(input: any, init?: RequestInit) { + // Handle URL input + if (typeof input === 'string' || input instanceof vmGlobalThis.URL) { + const urlString = String(input); + // Validate URL format + try { + new vmGlobalThis.URL(urlString); + this.url = urlString; + } catch (cause) { + throw new TypeError(`Failed to parse URL from ${urlString}`, { + cause, + }); + } + } else { + // Input is a Request object - clone its properties + this.url = input.url; + if (!init) { this.method = input.method; this.headers = new vmGlobalThis.Headers(input.headers); this.body = input.body; @@ -653,540 +620,551 @@ function createWorkflowSession({ this.signal = input.signal; this.duplex = input.duplex; this.destination = input.destination; + return; } + // If init is provided, merge: use source properties, then override with init + // Copy all properties from the source Request first + this.method = input.method; + this.headers = new vmGlobalThis.Headers(input.headers); + this.body = input.body; + this.mode = input.mode; + this.credentials = input.credentials; + this.cache = input.cache; + this.redirect = input.redirect; + this.referrer = input.referrer; + this.referrerPolicy = input.referrerPolicy; + this.integrity = input.integrity; + this.keepalive = input.keepalive; + this.signal = input.signal; + this.duplex = input.duplex; + this.destination = input.destination; + } - // Override with init options if provided - // Set method - if (init?.method) { - this.method = init.method.toUpperCase(); - } else if (typeof this.method !== 'string') { - // Fallback to default for string input case - this.method = 'GET'; - } - - // Set headers - if (init?.headers) { - this.headers = new vmGlobalThis.Headers(init.headers); - } else if ( - typeof input === 'string' || - input instanceof vmGlobalThis.URL - ) { - // For string/URL input, create empty headers - this.headers = new vmGlobalThis.Headers(); - } + // Override with init options if provided + // Set method + if (init?.method) { + this.method = init.method.toUpperCase(); + } else if (typeof this.method !== 'string') { + // Fallback to default for string input case + this.method = 'GET'; + } - // Set other properties with init values or defaults - if (init?.mode !== undefined) { - this.mode = init.mode; - } else if (typeof this.mode !== 'string') { - this.mode = 'cors'; - } + // Set headers + if (init?.headers) { + this.headers = new vmGlobalThis.Headers(init.headers); + } else if ( + typeof input === 'string' || + input instanceof vmGlobalThis.URL + ) { + // For string/URL input, create empty headers + this.headers = new vmGlobalThis.Headers(); + } - if (init?.credentials !== undefined) { - this.credentials = init.credentials; - } else if (typeof this.credentials !== 'string') { - this.credentials = 'same-origin'; - } + // Set other properties with init values or defaults + if (init?.mode !== undefined) { + this.mode = init.mode; + } else if (typeof this.mode !== 'string') { + this.mode = 'cors'; + } - // `any` cast here because @types/node v22 does not yet have `cache` - if ((init as any)?.cache !== undefined) { - this.cache = (init as any).cache; - } else if (typeof this.cache !== 'string') { - this.cache = 'default'; - } + if (init?.credentials !== undefined) { + this.credentials = init.credentials; + } else if (typeof this.credentials !== 'string') { + this.credentials = 'same-origin'; + } - if (init?.redirect !== undefined) { - this.redirect = init.redirect; - } else if (typeof this.redirect !== 'string') { - this.redirect = 'follow'; - } + // `any` cast here because @types/node v22 does not yet have `cache` + if ((init as any)?.cache !== undefined) { + this.cache = (init as any).cache; + } else if (typeof this.cache !== 'string') { + this.cache = 'default'; + } - if (init?.referrer !== undefined) { - this.referrer = init.referrer; - } else if (typeof this.referrer !== 'string') { - this.referrer = 'about:client'; - } + if (init?.redirect !== undefined) { + this.redirect = init.redirect; + } else if (typeof this.redirect !== 'string') { + this.redirect = 'follow'; + } - if (init?.referrerPolicy !== undefined) { - this.referrerPolicy = init.referrerPolicy; - } else if (typeof this.referrerPolicy !== 'string') { - this.referrerPolicy = ''; - } + if (init?.referrer !== undefined) { + this.referrer = init.referrer; + } else if (typeof this.referrer !== 'string') { + this.referrer = 'about:client'; + } - if (init?.integrity !== undefined) { - this.integrity = init.integrity; - } else if (typeof this.integrity !== 'string') { - this.integrity = ''; - } + if (init?.referrerPolicy !== undefined) { + this.referrerPolicy = init.referrerPolicy; + } else if (typeof this.referrerPolicy !== 'string') { + this.referrerPolicy = ''; + } - if (init?.keepalive !== undefined) { - this.keepalive = init.keepalive; - } else if (typeof this.keepalive !== 'boolean') { - this.keepalive = false; - } + if (init?.integrity !== undefined) { + this.integrity = init.integrity; + } else if (typeof this.integrity !== 'string') { + this.integrity = ''; + } - if (init?.signal !== undefined) { - // @ts-expect-error - AbortSignal stub - this.signal = init.signal; - } else if (!this.signal) { - // @ts-expect-error - AbortSignal stub - this.signal = { aborted: false }; - } + if (init?.keepalive !== undefined) { + this.keepalive = init.keepalive; + } else if (typeof this.keepalive !== 'boolean') { + this.keepalive = false; + } - if (!this.duplex) { - this.duplex = 'half'; - } + if (init?.signal !== undefined) { + // @ts-expect-error - AbortSignal stub + this.signal = init.signal; + } else if (!this.signal) { + // @ts-expect-error - AbortSignal stub + this.signal = { aborted: false }; + } - if (!this.destination) { - this.destination = 'document'; - } + if (!this.duplex) { + this.duplex = 'half'; + } - const body = init?.body; + if (!this.destination) { + this.destination = 'document'; + } - // Validate that GET/HEAD methods don't have a body - if ( - body !== null && - body !== undefined && - (this.method === 'GET' || this.method === 'HEAD') - ) { - throw new TypeError(`Request with GET/HEAD method cannot have body.`); - } + const body = init?.body; - // Store the original BodyInit for serialization - if (body !== null && body !== undefined) { - // Create a "fake" ReadableStream that stores the original body - // This avoids doing async work during workflow replay - this.body = Object.create(vmGlobalThis.ReadableStream.prototype, { - [BODY_INIT_SYMBOL]: { - value: body, - writable: false, - }, - }); - } else { - this.body = null; - } + // Validate that GET/HEAD methods don't have a body + if ( + body !== null && + body !== undefined && + (this.method === 'GET' || this.method === 'HEAD') + ) { + throw new TypeError(`Request with GET/HEAD method cannot have body.`); } - clone(): Request { - ENOTSUP(); + // Store the original BodyInit for serialization + if (body !== null && body !== undefined) { + // Create a "fake" ReadableStream that stores the original body + // This avoids doing async work during workflow replay + this.body = Object.create(vmGlobalThis.ReadableStream.prototype, { + [BODY_INIT_SYMBOL]: { + value: body, + writable: false, + }, + }); + } else { + this.body = null; } + } - get bodyUsed() { - return false; - } + clone(): Request { + ENOTSUP(); + } - // TODO: implement these - blob!: () => Promise; - formData!: () => Promise; + get bodyUsed() { + return false; + } - arrayBuffer!: () => Promise; - json!: () => Promise; - text!: () => Promise; + // TODO: implement these + blob!: () => Promise; + formData!: () => Promise; - async bytes() { - return new Uint8Array(await this.arrayBuffer()); - } + arrayBuffer!: () => Promise; + json!: () => Promise; + text!: () => Promise; + + async bytes() { + return new Uint8Array(await this.arrayBuffer()); } - vmGlobalThis.Request = Request; - - Object.defineProperties(Request.prototype, { - arrayBuffer: { - value: useStep<[], ArrayBuffer>('__builtin_response_array_buffer'), - writable: true, - configurable: true, - }, - json: { - value: useStep<[], any>('__builtin_response_json'), - writable: true, - configurable: true, - }, - text: { - value: useStep<[], string>('__builtin_response_text'), - writable: true, - configurable: true, - }, - }); + } + vmGlobalThis.Request = Request; + + Object.defineProperties(Request.prototype, { + arrayBuffer: { + value: useStep<[], ArrayBuffer>('__builtin_response_array_buffer'), + writable: true, + configurable: true, + }, + json: { + value: useStep<[], any>('__builtin_response_json'), + writable: true, + configurable: true, + }, + text: { + value: useStep<[], string>('__builtin_response_text'), + writable: true, + configurable: true, + }, + }); - class Response implements globalThis.Response { - type!: globalThis.Response['type']; - url!: string; - status!: number; - statusText!: string; - body!: ReadableStream | null; - headers!: Headers; - redirected!: boolean; - - constructor(body?: any, init?: ResponseInit) { - this.status = init?.status ?? 200; - this.statusText = init?.statusText ?? ''; - this.headers = new vmGlobalThis.Headers(init?.headers); - this.type = 'default'; - this.url = ''; - this.redirected = false; - - // Validate that null-body status codes don't have a body - // Per HTTP spec: 204 (No Content), 205 (Reset Content), and 304 (Not Modified) - if ( - body !== null && - body !== undefined && - (this.status === 204 || this.status === 205 || this.status === 304) - ) { - throw new TypeError( - `Response constructor: Invalid response status code ${this.status}` - ); - } + class Response implements globalThis.Response { + type!: globalThis.Response['type']; + url!: string; + status!: number; + statusText!: string; + body!: ReadableStream | null; + headers!: Headers; + redirected!: boolean; + + constructor(body?: any, init?: ResponseInit) { + this.status = init?.status ?? 200; + this.statusText = init?.statusText ?? ''; + this.headers = new vmGlobalThis.Headers(init?.headers); + this.type = 'default'; + this.url = ''; + this.redirected = false; + + // Validate that null-body status codes don't have a body + // Per HTTP spec: 204 (No Content), 205 (Reset Content), and 304 (Not Modified) + if ( + body !== null && + body !== undefined && + (this.status === 204 || this.status === 205 || this.status === 304) + ) { + throw new TypeError( + `Response constructor: Invalid response status code ${this.status}` + ); + } - // Store the original BodyInit for serialization - if (body !== null && body !== undefined) { - // Create a "fake" ReadableStream that stores the original body - // This avoids doing async work during workflow replay - this.body = Object.create(vmGlobalThis.ReadableStream.prototype, { - [BODY_INIT_SYMBOL]: { - value: body, - writable: false, - }, - }); - } else { - this.body = null; - } + // Store the original BodyInit for serialization + if (body !== null && body !== undefined) { + // Create a "fake" ReadableStream that stores the original body + // This avoids doing async work during workflow replay + this.body = Object.create(vmGlobalThis.ReadableStream.prototype, { + [BODY_INIT_SYMBOL]: { + value: body, + writable: false, + }, + }); + } else { + this.body = null; } + } - // TODO: implement these - clone!: () => Response; - blob!: () => Promise; - formData!: () => Promise; + // TODO: implement these + clone!: () => Response; + blob!: () => Promise; + formData!: () => Promise; - get ok() { - return this.status >= 200 && this.status < 300; - } + get ok() { + return this.status >= 200 && this.status < 300; + } - get bodyUsed() { - return false; - } + get bodyUsed() { + return false; + } - arrayBuffer!: () => Promise; - json!: () => Promise; - text!: () => Promise; + arrayBuffer!: () => Promise; + json!: () => Promise; + text!: () => Promise; - async bytes() { - return new Uint8Array(await this.arrayBuffer()); - } + async bytes() { + return new Uint8Array(await this.arrayBuffer()); + } - static json(data: any, init?: ResponseInit): Response { - const body = JSON.stringify(data); - const headers = new vmGlobalThis.Headers(init?.headers); - if (!headers.has('content-type')) { - headers.set('content-type', 'application/json'); - } - return new Response(body, { ...init, headers }); + static json(data: any, init?: ResponseInit): Response { + const body = JSON.stringify(data); + const headers = new vmGlobalThis.Headers(init?.headers); + if (!headers.has('content-type')) { + headers.set('content-type', 'application/json'); } + return new Response(body, { ...init, headers }); + } - static error(): Response { - ENOTSUP(); - } + static error(): Response { + ENOTSUP(); + } - static redirect(url: string | URL, status: number = 302): Response { - // Validate status code - only specific redirect codes are allowed - if (![301, 302, 303, 307, 308].includes(status)) { - throw new RangeError( - `Invalid redirect status code: ${status}. Must be one of: 301, 302, 303, 307, 308` - ); - } + static redirect(url: string | URL, status: number = 302): Response { + // Validate status code - only specific redirect codes are allowed + if (![301, 302, 303, 307, 308].includes(status)) { + throw new RangeError( + `Invalid redirect status code: ${status}. Must be one of: 301, 302, 303, 307, 308` + ); + } - // Create response with Location header - const headers = new vmGlobalThis.Headers(); - headers.set('Location', String(url)); + // Create response with Location header + const headers = new vmGlobalThis.Headers(); + headers.set('Location', String(url)); - const response = Object.create(Response.prototype); - response.status = status; - response.statusText = ''; - response.headers = headers; - response.body = null; - response.type = 'default'; - response.url = ''; - response.redirected = false; + const response = Object.create(Response.prototype); + response.status = status; + response.statusText = ''; + response.headers = headers; + response.body = null; + response.type = 'default'; + response.url = ''; + response.redirected = false; - return response; - } + return response; } - vmGlobalThis.Response = Response; - - Object.defineProperties(Response.prototype, { - arrayBuffer: { - value: useStep<[], ArrayBuffer>('__builtin_response_array_buffer'), - writable: true, - configurable: true, - }, - json: { - value: useStep<[], any>('__builtin_response_json'), - writable: true, - configurable: true, - }, - text: { - value: useStep<[], string>('__builtin_response_text'), - writable: true, - configurable: true, - }, - }); + } + vmGlobalThis.Response = Response; + + Object.defineProperties(Response.prototype, { + arrayBuffer: { + value: useStep<[], ArrayBuffer>('__builtin_response_array_buffer'), + writable: true, + configurable: true, + }, + json: { + value: useStep<[], any>('__builtin_response_json'), + writable: true, + configurable: true, + }, + text: { + value: useStep<[], string>('__builtin_response_text'), + writable: true, + configurable: true, + }, + }); - class ReadableStream implements globalThis.ReadableStream { - constructor() { - ENOTSUP(); - } + class ReadableStream implements globalThis.ReadableStream { + constructor() { + ENOTSUP(); + } - get locked() { - return false; - } + get locked() { + return false; + } - cancel(): any { - ENOTSUP(); - } + cancel(): any { + ENOTSUP(); + } - getReader(): any { - ENOTSUP(); - } + getReader(): any { + ENOTSUP(); + } - pipeThrough(): any { - ENOTSUP(); - } + pipeThrough(): any { + ENOTSUP(); + } - pipeTo(): any { - ENOTSUP(); - } + pipeTo(): any { + ENOTSUP(); + } - tee(): any { - ENOTSUP(); - } + tee(): any { + ENOTSUP(); + } - values(): any { - ENOTSUP(); - } + values(): any { + ENOTSUP(); + } - static from(): any { - ENOTSUP(); - } + static from(): any { + ENOTSUP(); + } - [Symbol.asyncIterator](): any { - ENOTSUP(); - } + [Symbol.asyncIterator](): any { + ENOTSUP(); } - vmGlobalThis.ReadableStream = ReadableStream; + } + vmGlobalThis.ReadableStream = ReadableStream; - class WritableStream implements globalThis.WritableStream { - constructor() { - ENOTSUP(); - } + class WritableStream implements globalThis.WritableStream { + constructor() { + ENOTSUP(); + } - get locked() { - return false; - } + get locked() { + return false; + } - abort(): any { - ENOTSUP(); - } + abort(): any { + ENOTSUP(); + } - close(): any { - ENOTSUP(); - } + close(): any { + ENOTSUP(); + } - getWriter(): any { - ENOTSUP(); - } + getWriter(): any { + ENOTSUP(); } - vmGlobalThis.WritableStream = WritableStream; + } + vmGlobalThis.WritableStream = WritableStream; - class TransformStream implements globalThis.TransformStream { - readable: globalThis.ReadableStream; - writable: globalThis.WritableStream; + class TransformStream implements globalThis.TransformStream { + readable: globalThis.ReadableStream; + writable: globalThis.WritableStream; - constructor() { - ENOTSUP(); - } + constructor() { + ENOTSUP(); } - vmGlobalThis.TransformStream = TransformStream; - - // Eventually we'll probably want to provide our own `console` object, - // but for now we'll just expose the global one. - vmGlobalThis.console = globalThis.console; - - // HACK: propagate symbol needed for AI gateway usage - const SYMBOL_FOR_REQ_CONTEXT = Symbol.for('@vercel/request-context'); - // @ts-expect-error - `@types/node` says symbol is not valid, but it does work - vmGlobalThis[SYMBOL_FOR_REQ_CONTEXT] = (globalThis as any)[ - SYMBOL_FOR_REQ_CONTEXT - ]; - - // Get a reference to the user-defined workflow function. - // The filename parameter ensures stack traces show a meaningful name - // (e.g., "example/workflows/99_e2e.ts") instead of "evalmachine.". - const parsedName = parseWorkflowName(workflowRun.workflowName); - const filename = parsedName?.moduleSpecifier || workflowRun.workflowName; - - // Evaluate the workflow bundle against the fresh context using a - // process-wide cache of the compiled `vm.Script`. The bundle is the same - // string for every replay and every invocation in this process, and - // compilation is a pure function of `(code, filename)`, so reusing the - // compiled Script across replays is determinism-safe: it produces the same - // workflow function and the same `filename` source attribution as - // re-parsing the bundle every time, but skips the (expensive) re-parse. - // Evaluating the bundle registers every workflow on - // `globalThis.__private_workflows`; the trailing lookup expression then - // retrieves the requested workflow function. The lookup is evaluated as a - // separate cached Script under the same `filename`, so error stack frames - // still attribute to the workflow's source file (`remapErrorStack` keys on - // `filename`). The one behavioural difference from the previous - // single-combined-string approach is the *line number* of an error thrown - // by the lookup expression itself: it now reports line 1 of the lookup - // Script rather than the line just past the end of the bundle. That path - // is rare (it requires the lookup `?.get(...)` expression to throw) and - // does not affect the workflow function or replay determinism. - // All SDK globals are installed; pin the serialization-consulted - // intrinsics before any workflow code can run. - freezeSerializationIntrinsics(vmGlobalThis); - - runCachedWorkflowScript(workflowCode, filename, context); - const workflowFn = runCachedWorkflowScript( - `globalThis.__private_workflows?.get(${JSON.stringify(workflowRun.workflowName)})`, - filename, - context + } + vmGlobalThis.TransformStream = TransformStream; + + // Eventually we'll probably want to provide our own `console` object, + // but for now we'll just expose the global one. + vmGlobalThis.console = globalThis.console; + + // HACK: propagate symbol needed for AI gateway usage + const SYMBOL_FOR_REQ_CONTEXT = Symbol.for('@vercel/request-context'); + // @ts-expect-error - `@types/node` says symbol is not valid, but it does work + vmGlobalThis[SYMBOL_FOR_REQ_CONTEXT] = (globalThis as any)[ + SYMBOL_FOR_REQ_CONTEXT + ]; + + // Get a reference to the user-defined workflow function. + // The filename parameter ensures stack traces show a meaningful name + // (e.g., "example/workflows/99_e2e.ts") instead of "evalmachine.". + const parsedName = parseWorkflowName(workflowRun.workflowName); + const filename = parsedName?.moduleSpecifier || workflowRun.workflowName; + + // Evaluate the workflow bundle against the fresh context using a + // process-wide cache of the compiled `vm.Script`. The bundle is the same + // string for every replay and every invocation in this process, and + // compilation is a pure function of `(code, filename)`, so reusing the + // compiled Script across replays is determinism-safe: it produces the same + // workflow function and the same `filename` source attribution as + // re-parsing the bundle every time, but skips the (expensive) re-parse. + // Evaluating the bundle registers every workflow on + // `globalThis.__private_workflows`; the trailing lookup expression then + // retrieves the requested workflow function. The lookup is evaluated as a + // separate cached Script under the same `filename`, so error stack frames + // still attribute to the workflow's source file (`remapErrorStack` keys on + // `filename`). The one behavioural difference from the previous + // single-combined-string approach is the *line number* of an error thrown + // by the lookup expression itself: it now reports line 1 of the lookup + // Script rather than the line just past the end of the bundle. That path + // is rare (it requires the lookup `?.get(...)` expression to throw) and + // does not affect the workflow function or replay determinism. + // All SDK globals are installed; pin the serialization-consulted + // intrinsics before any workflow code can run. + freezeSerializationIntrinsics(vmGlobalThis); + + runCachedWorkflowScript(workflowCode, filename, context); + const workflowFn = runCachedWorkflowScript( + `globalThis.__private_workflows?.get(${JSON.stringify(workflowRun.workflowName)})`, + filename, + context + ); + + if (typeof workflowFn !== 'function') { + throw new WorkflowNotRegisteredError(workflowRun.workflowName); + } + + // Chain workflow argument hydration onto the promiseQueue so that the + // unconsumed event check (which waits for the queue to drain) doesn't + // fire during the async gap between run_started consumption and the + // workflow function subscribing its first step callbacks. + let args: unknown[] = []; + workflowContext.promiseQueue = workflowContext.promiseQueue.then(async () => { + const prepared = await replayPayloadCache.prepareWorkflowInput(workflowRun); + args = await hydrateWorkflowArguments( + workflowRun.input, + workflowRun.runId, + encryptionKey, + vmGlobalThis, + {}, + prepared ); + }); + await workflowContext.promiseQueue; + + const workflowExecution = (async (): Promise => { + return await workflowFn(...args); + })(); - if (typeof workflowFn !== 'function') { - throw new WorkflowNotRegisteredError(workflowRun.workflowName); + const failWorkflow = async (error: unknown): Promise => { + // Control-flow signals are handled by the runtime and do not mean the + // workflow has terminally failed. `onWorkflowError` already moved the + // state machine (e.g. to `replay` on divergence, so a later resume falls + // back instead of throwing) — leave it alone. + if (WorkflowSuspension.is(error) || ReplayDivergenceError.is(error)) { + throw error; } + state = { type: 'completed' }; - // Chain workflow argument hydration onto the promiseQueue so that the - // unconsumed event check (which waits for the queue to drain) doesn't - // fire during the async gap between run_started consumption and the - // workflow function subscribing its first step callbacks. - let args: unknown[] = []; - workflowContext.promiseQueue = workflowContext.promiseQueue.then( - async () => { - const prepared = - await replayPayloadCache.prepareWorkflowInput(workflowRun); - args = await hydrateWorkflowArguments( - workflowRun.input, - workflowRun.runId, - encryptionKey, - vmGlobalThis, - {}, - prepared - ); - } + await drainPendingQueueItems( + workflowRun.runId, + workflowContext.invocationsQueue, + vmGlobalThis, + workflowRun, + 'failed', + runReadyBarrier ); - await workflowContext.promiseQueue; - const workflowExecution = (async (): Promise => { - return await workflowFn(...args); - })(); + throw error; + }; - const failWorkflow = async (error: unknown): Promise => { - state = { type: 'completed' }; - // Control-flow signals are handled by the runtime and do not mean the - // workflow has terminally failed. - if (WorkflowSuspension.is(error) || ReplayDivergenceError.is(error)) { + const waitForExecution = async ( + interruption: PromiseWithResolvers + ): Promise => { + let result: unknown; + try { + result = await Promise.race([workflowExecution, interruption.promise]); + } catch (error) { + if (state.type === 'suspended' && error === state.suspension) { throw error; } + return failWorkflow(error); + } + + state = { type: 'completed' }; + try { + const output = await dehydrateWorkflowReturnValue( + result, + workflowRun.runId, + encryptionKey, + vmGlobalThis, + false, + // Gate payload compression on the run's specVersion: only runs + // marked as possibly containing compressed payloads (spec >= 5) + // get gzip data. + (workflowRun.specVersion ?? 0) >= SPEC_VERSION_SUPPORTS_COMPRESSION + ); await drainPendingQueueItems( workflowRun.runId, workflowContext.invocationsQueue, vmGlobalThis, workflowRun, - 'failed', + 'completed', runReadyBarrier ); - throw error; - }; - - const completeWorkflow = async ( - result: unknown - ): Promise => { - state = { type: 'completed' }; - try { - const output = await dehydrateWorkflowReturnValue( - result, - workflowRun.runId, - encryptionKey, - vmGlobalThis, - false, - // Gate payload compression on the run's specVersion: only runs - // marked as possibly containing compressed payloads (spec >= 5) - // get gzip data. - (workflowRun.specVersion ?? 0) >= SPEC_VERSION_SUPPORTS_COMPRESSION - ); - - await drainPendingQueueItems( - workflowRun.runId, - workflowContext.invocationsQueue, - vmGlobalThis, - workflowRun, - 'completed', - runReadyBarrier - ); - - return { output, resultType: typeof result }; - } catch (error) { - return failWorkflow(error); - } - }; - - const waitForExecution = async ( - interruption: PromiseWithResolvers - ): Promise => { - let result: unknown; - try { - result = await Promise.race([workflowExecution, interruption.promise]); - } catch (error) { - if (state.type === 'suspended' && error === state.suspension) { - throw error; - } - return failWorkflow(error); - } - return completeWorkflow(result); - }; + return { output, resultType: typeof result }; + } catch (error) { + return failWorkflow(error); + } + }; - const session: WorkflowSession = { - workflowRun, - argumentCount: args.length, - resume(nextEvents) { - switch (state.type) { - case 'suspended': { - const consumedEvents = eventsConsumer.events; - const isStrictExtension = - nextEvents.length > consumedEvents.length && - consumedEvents.every( - (event, index) => event.eventId === nextEvents[index]?.eventId - ); - if (!isStrictExtension) { - state = { type: 'replay' }; - return { type: 'replay' }; - } - const interruption = withResolvers(); - state = { type: 'running', interruption }; - workflowContext.suspensionGeneration++; - eventsConsumer.append(nextEvents.slice(consumedEvents.length)); - return { - type: 'resumed', - execution: waitForExecution(interruption), - }; - } - case 'replay': - return { type: 'replay' }; - case 'completed': - case 'running': - throw new WorkflowRuntimeError( - `Cannot resume ${state.type} workflow "${workflowRun.runId}"` + const session: WorkflowSession = { + workflowRun, + argumentCount: args.length, + resume(nextEvents) { + switch (state.type) { + case 'suspended': { + const consumedEvents = eventsConsumer.events; + const isStrictExtension = + nextEvents.length > consumedEvents.length && + consumedEvents.every( + (event, index) => event.eventId === nextEvents[index]?.eventId ); + if (!isStrictExtension) { + state = { type: 'replay' }; + return { type: 'replay' }; + } + const interruption = withResolvers(); + state = { type: 'running', interruption }; + workflowContext.suspensionGeneration++; + eventsConsumer.append(nextEvents.slice(consumedEvents.length)); + return { + type: 'resumed', + execution: waitForExecution(interruption), + }; } - state satisfies never; - }, - }; + case 'replay': + return { type: 'replay' }; + case 'completed': + case 'running': + throw new WorkflowRuntimeError( + `Cannot resume ${state.type} workflow "${workflowRun.runId}"` + ); + } + state satisfies never; + }, + }; - return { - session, - execution: waitForExecution(initialInterruption), - }; - })(); + return { + session, + execution: waitForExecution(initialInterruption), + }; } From c61f9888190e9e003e1ab4e7f173a6f44c4577b8 Mon Sep 17 00:00:00 2001 From: Nathan Colosimo <110621881+NathanColosimo@users.noreply.github.com> Date: Wed, 22 Jul 2026 23:22:00 -0700 Subject: [PATCH 30/43] chore: retrigger vercel deployments From 9395b2d2ae9aa328bbdcc734f221c0c276f140aa Mon Sep 17 00:00:00 2001 From: Nathan Colosimo <110621881+NathanColosimo@users.noreply.github.com> Date: Thu, 23 Jul 2026 15:17:48 -0700 Subject: [PATCH 31/43] Drop serialization intrinsic freezing from the sandbox The retained-VM passivity design moved from pinning/verifying the sandbox surfaces serialization dispatches on to injecting hardened operations into devalue itself (with taint-based de-opt), so freezing Object/Array/Function prototypes and pinning global bindings is no longer needed. Keep only the determinism hardening (sync digest, removal of wall-clock/GC-observing APIs). --- .changeset/vm-determinism-hardening.md | 2 +- .../v5/api-reference/workflow-globals.mdx | 1 - packages/core/src/vm/index.test.ts | 74 +--------------- packages/core/src/vm/index.ts | 84 ------------------- packages/core/src/workflow.ts | 6 +- 5 files changed, 3 insertions(+), 164 deletions(-) diff --git a/.changeset/vm-determinism-hardening.md b/.changeset/vm-determinism-hardening.md index d40a2cffd8..912375fc39 100644 --- a/.changeset/vm-determinism-hardening.md +++ b/.changeset/vm-determinism-hardening.md @@ -3,4 +3,4 @@ 'workflow': patch --- -Deterministic sandbox hardening: `crypto.subtle.digest` in workflow functions now computes synchronously via `node:crypto` (byte-identical values, deterministic timing under replay); `Atomics.waitAsync`, async `WebAssembly` compilation, `WeakRef`, and `FinalizationRegistry` are no longer exposed (wall-clock timing and GC observation cannot be replayed); and the sandbox pins the surfaces serialization dispatch resolves through: `Object.prototype`/`Array.prototype`/`Function.prototype` are frozen and serialization-relevant global bindings are non-writable. Built-in prototypes and constructor statics remain patchable, so polyfills (Temporal, core-js) keep working. +Deterministic sandbox hardening: `crypto.subtle.digest` in workflow functions now computes synchronously via `node:crypto` (byte-identical values, deterministic timing under replay); `Atomics.waitAsync`, async `WebAssembly` compilation, `WeakRef`, and `FinalizationRegistry` are no longer exposed (wall-clock timing and GC observation cannot be replayed). diff --git a/docs/content/docs/v5/api-reference/workflow-globals.mdx b/docs/content/docs/v5/api-reference/workflow-globals.mdx index 2ec290738f..6faa7e3df6 100644 --- a/docs/content/docs/v5/api-reference/workflow-globals.mdx +++ b/docs/content/docs/v5/api-reference/workflow-globals.mdx @@ -103,4 +103,3 @@ The following are **not available** in workflow functions. Move this logic to [s - **`WeakRef` and `FinalizationRegistry`**: garbage-collection timing is not deterministic, so observing it would make workflow code impossible to replay. (`WeakMap` and `WeakSet` remain available — they do not expose GC state.) - **`Atomics.waitAsync`**: a wall-clock timer, which cannot be replayed. Use [`sleep()`](/docs/api-reference/workflow/sleep) instead. - **Async `WebAssembly` compilation** (`compile`, `instantiate`, `compileStreaming`, `instantiateStreaming`): resolves on compile-thread timing. The synchronous `new WebAssembly.Module()` and `new WebAssembly.Instance()` constructors remain available. -- **Mutating core intrinsics**: `Object.prototype`, `Array.prototype`, and `Function.prototype` are frozen, and the global bindings for serialization-relevant built-ins (`Map`, `Date`, `Headers`, and similar) cannot be reassigned — serialization dispatch resolves through these, so redefining them would make replay unfaithful. Adding methods to built-in prototypes (e.g. a Temporal polyfill's `Date.prototype.toTemporalInstant`) and polyfilling constructor statics (`Object.groupBy`) remain allowed. diff --git a/packages/core/src/vm/index.test.ts b/packages/core/src/vm/index.test.ts index f28f330cb4..7e778e10e1 100644 --- a/packages/core/src/vm/index.test.ts +++ b/packages/core/src/vm/index.test.ts @@ -1,6 +1,6 @@ import * as vm from 'node:vm'; import { describe, expect, it } from 'vitest'; -import { createContext, freezeSerializationIntrinsics } from './index.js'; +import { createContext } from './index.js'; const seed = 'entropy seed'; const fixedTimestamp = 1234567890000; @@ -404,75 +404,3 @@ describe('crypto.subtle.digest view metadata', () => { expect(new Uint8Array(result)).toEqual(new Uint8Array(expected)); }); }); - -describe('freezeSerializationIntrinsics', () => { - it('freezes the universal lookup backstops', () => { - const { context, globalThis: g } = createContext({ seed, fixedTimestamp }); - freezeSerializationIntrinsics(g); - - expect(() => - vm.runInContext(`"use strict"; Object.prototype.polluted = 1`, context) - ).toThrow(/not extensible/); - expect(() => - vm.runInContext( - `"use strict"; Object.defineProperty(Array.prototype, "signal", { get() { return 1; } })`, - context - ) - ).toThrow(/not extensible/); - expect(() => - vm.runInContext( - `"use strict"; Function.prototype[Symbol.hasInstance] = () => true`, - context - ) - ).toThrow(/read only|not extensible/); - }); - - it('pins global bindings, including intentionally absent ones', () => { - const { context, globalThis: g } = createContext({ seed, fixedTimestamp }); - freezeSerializationIntrinsics(g); - - vm.runInContext( - 'try { globalThis.Map = function () {} } catch {}', - context - ); - expect(vm.runInContext('typeof Map.prototype.get', context)).toBe( - 'function' - ); - vm.runInContext( - 'try { globalThis.Request = function () {} } catch {}', - context - ); - expect(vm.runInContext('typeof Request', context)).toBe('undefined'); - }); - - it('leaves value-type prototypes and constructor statics patchable (polyfills)', () => { - const { context, globalThis: g } = createContext({ seed, fixedTimestamp }); - freezeSerializationIntrinsics(g); - - // Temporal-style prototype method and modern static polyfills must work. - const result = vm.runInContext( - `"use strict"; - Date.prototype.toTemporalInstant = function () { return "instant:" + this.getTime(); }; - Object.groupBy = (items, fn) => items.reduce((acc, x) => ((acc[fn(x)] ??= []).push(x), acc), {}); - Set.prototype.union = function (other) { return new Set([...this, ...other]); }; - [ - new Date(5).toTemporalInstant(), - Object.groupBy([1, 2, 3], (x) => (x % 2 ? "odd" : "even")).odd.length, - new Set([1]).union(new Set([2])).size, - ]`, - context - ); - expect(result[0]).toBe('instant:5'); - expect(result[1]).toBe(2); - expect(result[2]).toBe(2); - }); - - it('leaves ordinary workflow globals writable', () => { - const { context, globalThis: g } = createContext({ seed, fixedTimestamp }); - freezeSerializationIntrinsics(g); - - expect( - vm.runInContext('globalThis.myState = { ok: true }; myState.ok', context) - ).toBe(true); - }); -}); diff --git a/packages/core/src/vm/index.ts b/packages/core/src/vm/index.ts index f23bb71488..39386d0378 100644 --- a/packages/core/src/vm/index.ts +++ b/packages/core/src/vm/index.ts @@ -75,90 +75,6 @@ function toDigestBytes(data: ArrayBuffer | ArrayBufferView): Uint8Array { return new Uint8Array(data as ArrayBuffer); } -// Global bindings the serialization reducers dispatch on (`value instanceof -// global.X` — e.g. the Map/Set/Headers reducers in -// packages/core/src/serialization/reducers/common.ts:312-329 and the -// Request/Response/stream reducers in packages/core/src/serialization.ts). -// Pinned (non-writable) so workflow code cannot swap in a constructor whose -// `Symbol.hasInstance` the serializer would then execute. Absent bindings are -// pinned to `undefined` so a spoofing global cannot be introduced either. -const SERIALIZATION_BINDINGS = [ - 'Object', - 'Array', - 'Function', - 'Map', - 'Set', - 'Date', - 'RegExp', - 'ArrayBuffer', - 'SharedArrayBuffer', - 'DataView', - 'Int8Array', - 'Uint8Array', - 'Uint8ClampedArray', - 'Int16Array', - 'Uint16Array', - 'Int32Array', - 'Uint32Array', - 'Float16Array', - 'Float32Array', - 'Float64Array', - 'BigInt64Array', - 'BigUint64Array', - 'Headers', - 'URL', - 'URLSearchParams', - 'DOMException', - 'AbortController', - 'AbortSignal', - 'Request', - 'Response', - 'ReadableStream', - 'WritableStream', - 'TransformStream', -] as const; - -/** - * Pin the sandbox surfaces that serialization dispatch resolves through. - * Called after ALL SDK globals are installed, immediately before the - * workflow bundle evaluates. - * - * Frozen — the universal lookup backstops only: - * - `Object.prototype` / `Array.prototype`: every missed property read on a - * plain object or array terminates here (reducer probes like `.signal`, - * `Symbol.toStringTag` lookups by devalue's type detection, `constructor` - * reads by the class reducer in - * packages/core/src/serialization/reducers/class.ts:26-43), so a planted - * accessor here would execute during serialization of ANY value. - * - `Function.prototype`: `x instanceof C` resolves `Symbol.hasInstance` - * through the constructor's prototype chain (tc39.es/ecma262/#sec-instanceofoperator), - * which ends here for every constructor. - * No real-world polyfill patches these three, so freezing them costs nothing. - * - * NOT frozen — constructors and the value-type prototypes (`Map.prototype`, - * `Date.prototype`, typed arrays, …): polyfills legitimately add methods - * there (Temporal's `Date.prototype.toTemporalInstant`, core-js - * `Set.prototype.union`, `Object.groupBy`, `Array.fromAsync`). New - * data-valued members are inert during serialization — only the specific - * members the serializer EXECUTES matter, and the retained-input gate - * verifies those per boundary against captured originals instead - * (see runtime/retained-step-input.ts). - */ -export function freezeSerializationIntrinsics(g: typeof globalThis): void { - Object.freeze(g.Object.prototype); - Object.freeze(g.Array.prototype); - Object.freeze(g.Function.prototype); - for (const name of SERIALIZATION_BINDINGS) { - const descriptor = Object.getOwnPropertyDescriptor(g, name); - Object.defineProperty(g, name, { - value: descriptor && 'value' in descriptor ? descriptor.value : undefined, - writable: false, - configurable: false, - enumerable: descriptor?.enumerable ?? false, - }); - } -} - /** * Creates a Node.js `vm.Context` configured to be usable for * executing workflow logic in a deterministic environment. diff --git a/packages/core/src/workflow.ts b/packages/core/src/workflow.ts index 8ed18de3d9..06fe1d79e2 100644 --- a/packages/core/src/workflow.ts +++ b/packages/core/src/workflow.ts @@ -38,7 +38,7 @@ import { import * as Attribute from './telemetry/semantic-conventions.js'; import { trace } from './telemetry.js'; import { getWorkflowRunStreamId } from './util.js'; -import { createContext, freezeSerializationIntrinsics } from './vm/index.js'; +import { createContext } from './vm/index.js'; import { runCachedWorkflowScript } from './vm/script-cache.js'; import { createAbortSignalStatics, @@ -847,10 +847,6 @@ export async function runWorkflow( // Script rather than the line just past the end of the bundle. That path // is rare (it requires the lookup `?.get(...)` expression to throw) and // does not affect the workflow function or replay determinism. - // All SDK globals are installed; pin the serialization-consulted - // intrinsics before any workflow code can run. - freezeSerializationIntrinsics(vmGlobalThis); - runCachedWorkflowScript(workflowCode, filename, context); const workflowFn = runCachedWorkflowScript( `globalThis.__private_workflows?.get(${JSON.stringify(workflowRun.workflowName)})`, From 8e23cfafbe008250aa202f51ad2c5b02f361d1a9 Mon Sep 17 00:00:00 2001 From: Nathan Colosimo <110621881+NathanColosimo@users.noreply.github.com> Date: Thu, 23 Jul 2026 16:25:55 -0700 Subject: [PATCH 32/43] Document and lock in why async crypto.subtle methods cannot break quiescence MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The remaining async subtle methods reject immediately through the crypto proxy (brand check — the receiver is not a real SubtleCrypto), so they can never mint a host-timing promise. Narrow the quiescence comment to what the code actually enforces and add a test so the unreachability is not silently "fixed" later. --- packages/core/src/vm/index.test.ts | 18 ++++++++++++++++++ packages/core/src/vm/index.ts | 12 +++++++++++- 2 files changed, 29 insertions(+), 1 deletion(-) diff --git a/packages/core/src/vm/index.test.ts b/packages/core/src/vm/index.test.ts index 7e778e10e1..a2a9d5627d 100644 --- a/packages/core/src/vm/index.test.ts +++ b/packages/core/src/vm/index.test.ts @@ -171,6 +171,24 @@ describe('createContext', () => { expect(err?.message).toContain('not available inside a workflow function'); }); + it('rejects the remaining async `crypto.subtle` methods immediately', async () => { + // These would settle on host threadpool timing (breaking the quiescence + // invariant a retained VM relies on), so they must stay unreachable: + // through the crypto proxy the receiver is not a real SubtleCrypto and + // the brand check rejects before any crypto work is scheduled. + const { context } = createContext({ seed, fixedTimestamp }); + for (const call of [ + 'crypto.subtle.importKey("raw", new Uint8Array(16), { name: "AES-GCM" }, false, ["encrypt"])', + 'crypto.subtle.encrypt({ name: "AES-GCM" }, {}, new Uint8Array(4))', + 'crypto.subtle.sign("HMAC", {}, new Uint8Array(4))', + 'crypto.subtle.deriveBits({ name: "PBKDF2" }, {}, 128)', + ]) { + await expect(vm.runInContext(call, context)).rejects.toThrow( + /must be of type SubtleCrypto/ + ); + } + }); + it('should call `onWorkflowError` when a workflow error occurs', async () => { const { context } = createContext({ seed, fixedTimestamp }); diff --git a/packages/core/src/vm/index.ts b/packages/core/src/vm/index.ts index 39386d0378..59a6bda589 100644 --- a/packages/core/src/vm/index.ts +++ b/packages/core/src/vm/index.ts @@ -127,7 +127,7 @@ export function createContext(options: CreateContextOptions) { // timing or host state: after this block, every promise a workflow can // create settles either from the event log or within its own microtask // cascade, so a suspended VM is fully quiescent and can be retained across - // inline steps (see `canRetainWorkflowSession`). + // inline steps. // // - `Atomics.waitAsync` is a wall-clock timer (via SharedArrayBuffer). // - The async `WebAssembly` entry points resolve on compile-thread timing; @@ -135,6 +135,16 @@ export function createContext(options: CreateContextOptions) { // - `WeakRef.deref()` and finalizer callbacks observe GC timing. // - Dynamic `import()` settles within a microtask (rejected: no // `importModuleDynamically`), so it needs no handling. + // - The remaining async `crypto.subtle` methods (`encrypt`, `sign`, + // `importKey`, …) would settle on host threadpool timing, but none of + // them are usable: invoked through the crypto proxy below the receiver + // is not a real SubtleCrypto, so the brand check rejects immediately + // ("Value of 'this' must be of type SubtleCrypto") before any crypto + // work is scheduled — a deterministic microtask rejection. Only the + // deterministic overrides (`digest`, `getRandomValues`, `randomUUID`) + // and the explicit `generateKey` error do real work — locked in by a + // test. Do not "fix" those methods by binding them to the host subtle: + // that would hand workflows a host-timing promise. const intrinsics = g as unknown as Record>; delete intrinsics.Atomics.waitAsync; delete intrinsics.WebAssembly.compile; From 9abf488709b0b1340e4237e966beeb8a98a960e5 Mon Sep 17 00:00:00 2001 From: Nathan Colosimo <110621881+NathanColosimo@users.noreply.github.com> Date: Fri, 24 Jul 2026 14:13:44 -0700 Subject: [PATCH 33/43] simplify sandbox hardening: lean digest input conversion, async digest, explicit subtle throwers --- .changeset/vm-determinism-hardening.md | 4 +- packages/core/src/vm/index.test.ts | 42 +++----- packages/core/src/vm/index.ts | 132 +++++++++---------------- 3 files changed, 64 insertions(+), 114 deletions(-) diff --git a/.changeset/vm-determinism-hardening.md b/.changeset/vm-determinism-hardening.md index 912375fc39..8c3bbdd6a2 100644 --- a/.changeset/vm-determinism-hardening.md +++ b/.changeset/vm-determinism-hardening.md @@ -1,6 +1,6 @@ --- -'@workflow/core': patch -'workflow': patch +'@workflow/core': minor +'workflow': minor --- Deterministic sandbox hardening: `crypto.subtle.digest` in workflow functions now computes synchronously via `node:crypto` (byte-identical values, deterministic timing under replay); `Atomics.waitAsync`, async `WebAssembly` compilation, `WeakRef`, and `FinalizationRegistry` are no longer exposed (wall-clock timing and GC observation cannot be replayed). diff --git a/packages/core/src/vm/index.test.ts b/packages/core/src/vm/index.test.ts index a2a9d5627d..3f2ebbabeb 100644 --- a/packages/core/src/vm/index.test.ts +++ b/packages/core/src/vm/index.test.ts @@ -171,11 +171,9 @@ describe('createContext', () => { expect(err?.message).toContain('not available inside a workflow function'); }); - it('rejects the remaining async `crypto.subtle` methods immediately', async () => { + it('throws for the remaining async `crypto.subtle` methods', () => { // These would settle on host threadpool timing (breaking the quiescence - // invariant a retained VM relies on), so they must stay unreachable: - // through the crypto proxy the receiver is not a real SubtleCrypto and - // the brand check rejects before any crypto work is scheduled. + // invariant a retained VM relies on), so they must stay unreachable. const { context } = createContext({ seed, fixedTimestamp }); for (const call of [ 'crypto.subtle.importKey("raw", new Uint8Array(16), { name: "AES-GCM" }, false, ["encrypt"])', @@ -183,8 +181,8 @@ describe('createContext', () => { 'crypto.subtle.sign("HMAC", {}, new Uint8Array(4))', 'crypto.subtle.deriveBits({ name: "PBKDF2" }, {}, 128)', ]) { - await expect(vm.runInContext(call, context)).rejects.toThrow( - /must be of type SubtleCrypto/ + expect(() => vm.runInContext(call, context)).toThrow( + /not available inside a workflow function/ ); } }); @@ -300,6 +298,18 @@ describe('host-timed async', () => { 'function' ); }); + + it('does not expose WeakRef or FinalizationRegistry', () => { + const { context } = createContext({ seed, fixedTimestamp }); + + expect(vm.runInContext('typeof WeakRef', context)).toBe('undefined'); + expect(vm.runInContext('typeof FinalizationRegistry', context)).toBe( + 'undefined' + ); + // WeakMap/WeakSet do not expose GC state and stay available. + expect(vm.runInContext('typeof WeakMap', context)).toBe('function'); + expect(vm.runInContext('typeof WeakSet', context)).toBe('function'); + }); }); describe('crypto.subtle.digest', () => { @@ -358,23 +368,7 @@ describe('crypto.subtle.digest', () => { vm.runInContext('crypto.subtle.digest("MD5", new Uint8Array(1))', context) ).rejects.toMatchObject({ name: 'NotSupportedError' }); }); -}); - -describe('GC observation', () => { - it('does not expose WeakRef or FinalizationRegistry', () => { - const { context } = createContext({ seed, fixedTimestamp }); - - expect(vm.runInContext('typeof WeakRef', context)).toBe('undefined'); - expect(vm.runInContext('typeof FinalizationRegistry', context)).toBe( - 'undefined' - ); - // WeakMap/WeakSet do not expose GC state and stay available. - expect(vm.runInContext('typeof WeakMap', context)).toBe('function'); - expect(vm.runInContext('typeof WeakSet', context)).toBe('function'); - }); -}); -describe('crypto.subtle.digest input validation', () => { it.each([ ['a number', '2000000000'], ['a plain object', '({})'], @@ -387,9 +381,7 @@ describe('crypto.subtle.digest input validation', () => { vm.runInContext(`crypto.subtle.digest("SHA-256", ${expression})`, context) ).rejects.toThrow(TypeError); }); -}); -describe('crypto.subtle.digest SharedArrayBuffer rejection', () => { it('rejects SharedArrayBuffer-backed views like WebCrypto does', async () => { const { context } = createContext({ seed, fixedTimestamp }); @@ -400,9 +392,7 @@ describe('crypto.subtle.digest SharedArrayBuffer rejection', () => { ) ).rejects.toThrow(TypeError); }); -}); -describe('crypto.subtle.digest view metadata', () => { it('reads view ranges from internal slots, ignoring shadowed properties', async () => { const { context } = createContext({ seed, fixedTimestamp }); diff --git a/packages/core/src/vm/index.ts b/packages/core/src/vm/index.ts index 59a6bda589..0355abb884 100644 --- a/packages/core/src/vm/index.ts +++ b/packages/core/src/vm/index.ts @@ -20,59 +20,26 @@ const DIGEST_ALGORITHMS: Record = { 'SHA-512': 'sha512', }; -// Intrinsic prototype getters, captured so view metadata is read from -// internal slots like WebCrypto's BufferSource conversion — own properties -// shadowing `buffer`/`byteOffset`/`byteLength` on a view must not change -// which bytes are hashed. -function intrinsicGetter(prototype: object, name: string) { - // biome-ignore lint/style/noNonNullAssertion: intrinsic accessors always exist - return Object.getOwnPropertyDescriptor(prototype, name)!.get!; -} -const arrayBufferByteLength = intrinsicGetter( - ArrayBuffer.prototype, - 'byteLength' -); -const typedArrayPrototype = Object.getPrototypeOf( - Uint8Array.prototype -) as object; -const viewGetters = { - typedArray: { - buffer: intrinsicGetter(typedArrayPrototype, 'buffer'), - byteOffset: intrinsicGetter(typedArrayPrototype, 'byteOffset'), - byteLength: intrinsicGetter(typedArrayPrototype, 'byteLength'), - }, - dataView: { - buffer: intrinsicGetter(DataView.prototype, 'buffer'), - byteOffset: intrinsicGetter(DataView.prototype, 'byteOffset'), - byteLength: intrinsicGetter(DataView.prototype, 'byteLength'), - }, -}; - -// WebCrypto BufferSource conversion: typed-array/DataView views are read via -// internal slots; anything else must be a real ArrayBuffer (the native -// byteLength getter is a brand check that works across vm realms) so that -// e.g. a plain number is rejected with TypeError instead of allocating a -// Uint8Array of that length. -function toDigestBytes(data: ArrayBuffer | ArrayBufferView): Uint8Array { +// WebCrypto BufferSource conversion. The `util.types` brand checks work +// across vm realms, and node:crypto's `hash.update()` reads view ranges from +// internal slots (own properties shadowing `byteOffset`/`byteLength` are +// ignored) — matching WebCrypto, which also reads internal slots. +function toDigestInput( + data: ArrayBuffer | ArrayBufferView +): NodeJS.ArrayBufferView { if (types.isTypedArray(data) || types.isDataView(data)) { - const getters = types.isDataView(data) - ? viewGetters.dataView - : viewGetters.typedArray; - const buffer = getters.buffer.call(data) as ArrayBuffer; // WebCrypto's BufferSource excludes SharedArrayBuffer-backed views. - if (types.isSharedArrayBuffer(buffer)) { + if (types.isSharedArrayBuffer(data.buffer)) { throw new TypeError( 'crypto.subtle.digest does not accept SharedArrayBuffer-backed views' ); } - return new Uint8Array( - buffer, - getters.byteOffset.call(data) as number, - getters.byteLength.call(data) as number - ); + return data; + } + if (!types.isArrayBuffer(data)) { + throw new TypeError('data must be an ArrayBuffer or ArrayBufferView'); } - arrayBufferByteLength.call(data); - return new Uint8Array(data as ArrayBuffer); + return new Uint8Array(data); } /** @@ -123,11 +90,11 @@ export function createContext(options: CreateContextOptions) { const randomUUID = createRandomUUID(rng); - // The sandbox must not expose any way for workflow code to observe host - // timing or host state: after this block, every promise a workflow can - // create settles either from the event log or within its own microtask - // cascade, so a suspended VM is fully quiescent and can be retained across - // inline steps. + // The sandbox must not hand workflow code a promise that settles on host + // timing or a way to observe host state (best-effort, for non-adversarial + // code): every promise a workflow creates settles either from the event log + // or within its own microtask cascade, so a suspended VM is fully quiescent + // and can be retained across inline steps. // // - `Atomics.waitAsync` is a wall-clock timer (via SharedArrayBuffer). // - The async `WebAssembly` entry points resolve on compile-thread timing; @@ -135,16 +102,9 @@ export function createContext(options: CreateContextOptions) { // - `WeakRef.deref()` and finalizer callbacks observe GC timing. // - Dynamic `import()` settles within a microtask (rejected: no // `importModuleDynamically`), so it needs no handling. - // - The remaining async `crypto.subtle` methods (`encrypt`, `sign`, - // `importKey`, …) would settle on host threadpool timing, but none of - // them are usable: invoked through the crypto proxy below the receiver - // is not a real SubtleCrypto, so the brand check rejects immediately - // ("Value of 'this' must be of type SubtleCrypto") before any crypto - // work is scheduled — a deterministic microtask rejection. Only the - // deterministic overrides (`digest`, `getRandomValues`, `randomUUID`) - // and the explicit `generateKey` error do real work — locked in by a - // test. Do not "fix" those methods by binding them to the host subtle: - // that would hand workflows a host-timing promise. + // - All `crypto.subtle` methods except the deterministic `digest` override + // below throw explicitly (see the subtle proxy) — binding them to the + // host subtle would hand workflows a host-timing promise. const intrinsics = g as unknown as Record>; delete intrinsics.Atomics.waitAsync; delete intrinsics.WebAssembly.compile; @@ -158,27 +118,23 @@ export function createContext(options: CreateContextOptions) { // promise settles on a deterministic microtask instead of host threadpool // timing — a digest can never advance a suspended workflow, and // digest-using VMs stay retainable. Values are byte-identical to WebCrypto. - const digest = ( + const digest = async ( algorithm: string | { name: string }, data: ArrayBuffer | ArrayBufferView ): Promise => { - try { - const name = typeof algorithm === 'string' ? algorithm : algorithm.name; - const ossl = DIGEST_ALGORITHMS[name.toUpperCase()]; - if (!ossl) { - throw new DOMException( - `Unrecognized algorithm name: ${name}`, - 'NotSupportedError' - ); - } - const hash = createHash(ossl).update(toDigestBytes(data)).digest(); - // Copy out: small Buffers share the internal pool allocation. - const out = new ArrayBuffer(hash.byteLength); - new Uint8Array(out).set(hash); - return Promise.resolve(out); - } catch (error) { - return Promise.reject(error); + const name = typeof algorithm === 'string' ? algorithm : algorithm.name; + const ossl = DIGEST_ALGORITHMS[name.toUpperCase()]; + if (!ossl) { + throw new DOMException( + `Unrecognized algorithm name: ${name}`, + 'NotSupportedError' + ); } + const hash = createHash(ossl).update(toDigestInput(data)).digest(); + // Copy out: small Buffers share the internal pool allocation. + const out = new ArrayBuffer(hash.byteLength); + new Uint8Array(out).set(hash); + return out; }; g.crypto = new Proxy(originalCrypto, { @@ -192,16 +148,20 @@ export function createContext(options: CreateContextOptions) { if (prop === 'subtle') { return new Proxy(originalSubtle, { get(target, prop) { - if (prop === 'generateKey') { - return () => { - throw new WorkflowRuntimeError( - '`crypto.subtle.generateKey()` is not available inside a workflow function. Move key generation to a step function where full Node.js crypto is available.' - ); - }; - } else if (prop === 'digest') { + if (prop === 'digest') { return digest; } - return target[prop as keyof typeof originalSubtle]; + const value = target[prop as keyof typeof originalSubtle]; + if (typeof value !== 'function') { + return value; + } + // Every other subtle method (`encrypt`, `sign`, `importKey`, …) + // settles on host threadpool timing, which cannot be replayed. + return () => { + throw new WorkflowRuntimeError( + `\`crypto.subtle.${String(prop)}()\` is not available inside a workflow function. Move it to a step function where full Node.js crypto is available.` + ); + }; }, }); } From 3cd4d0747605f4c92731990d2490710a58782fb8 Mon Sep 17 00:00:00 2001 From: Nathan Colosimo <110621881+NathanColosimo@users.noreply.github.com> Date: Fri, 24 Jul 2026 14:23:16 -0700 Subject: [PATCH 34/43] simplify retention: single decision site in suspension catch, steps-only allow-list gate, drop prepareForRetention param --- .changeset/retain-workflow-vm.md | 4 +- packages/core/src/runtime.ts | 88 +++++++++---------- .../core/src/runtime/suspension-handler.ts | 27 ++---- packages/core/src/workflow.ts | 9 +- 4 files changed, 59 insertions(+), 69 deletions(-) diff --git a/.changeset/retain-workflow-vm.md b/.changeset/retain-workflow-vm.md index 25d38b0d57..718282d550 100644 --- a/.changeset/retain-workflow-vm.md +++ b/.changeset/retain-workflow-vm.md @@ -1,6 +1,6 @@ --- -'@workflow/core': patch -'workflow': patch +'@workflow/core': minor +'workflow': minor --- Retain workflow execution across inline steps within one invocation; `WORKFLOW_RETAINED_VM=0` disables retention. Boundaries whose step arguments are not primitive values fall back to ordinary replay. diff --git a/packages/core/src/runtime.ts b/packages/core/src/runtime.ts index 3c6ae0d375..b30a9568d0 100644 --- a/packages/core/src/runtime.ts +++ b/packages/core/src/runtime.ts @@ -361,9 +361,11 @@ function openHookAndWaitState(events: Event[]): { } /** - * Retain only pure step boundaries with no out-of-band continuation source. - * Attributes require replay; hooks and waits can wake another invocation. - * `WORKFLOW_RETAINED_VM=0` disables retention entirely. + * Retain only pure step boundaries (every suspension item is a step — any + * other item type, present or future, is unretainable by default) with no + * out-of-band continuation source: attributes require replay; hooks and + * waits can wake another invocation. `WORKFLOW_RETAINED_VM=0` disables + * retention entirely. * * Quiescence assumes workflow code stays inside the sandbox's determinism * contract. Escaping to the host realm (e.g. recovering the host `Function` @@ -373,21 +375,15 @@ function openHookAndWaitState(events: Event[]): { */ function canRetainWorkflowSession( suspension: WorkflowSuspension, - events: Event[] + openHookWaitState: { openHook: boolean; openWait: boolean } ): boolean { - if ( - !isVmRetentionEnabled() || - suspension.stepCount === 0 || - suspension.hookCount > 0 || - suspension.waitCount > 0 || - suspension.attributeCount > 0 || - suspension.hookDisposedCount > 0 || - suspension.abortCount > 0 - ) { - return false; - } - const { openHook, openWait } = openHookAndWaitState(events); - return !openHook && !openWait; + return ( + isVmRetentionEnabled() && + suspension.steps.length > 0 && + suspension.steps.every((item) => item.type === 'step') && + !openHookWaitState.openHook && + !openHookWaitState.openWait + ); } /** @@ -1603,12 +1599,11 @@ export function workflowEntrypoint( // point and the inline executeStep mutates eventsCursor. preInlineWriteCursor = eventsCursor; - let executionMode = workflowExecution.type; runtimeLogger.debug('Starting workflow execution', { workflowRunId: runId, loopIteration, eventCount: events.length, - executionMode, + executionMode: workflowExecution.type, }); replayStart = Date.now(); let workflowResult: WorkflowExecutionResult = @@ -1621,7 +1616,6 @@ export function workflowEntrypoint( : { type: 'replay' }; if (workflowResult.type === 'replay') { - executionMode = 'replay'; workflowExecution = { type: 'replay' }; // Start every missing decrypt/decompress operation // before VM setup. Web Crypto work can overlap bundle @@ -1648,15 +1642,13 @@ export function workflowEntrypoint( } if (workflowResult.type === 'suspended') { - workflowExecution = canRetainWorkflowSession( - workflowResult.suspension, - events - ) - ? { - type: 'retained', - session: workflowResult.session, - } - : { type: 'replay' }; + // Park the live session; the suspension catch below + // makes the one retention decision — keep it for the + // next iteration or discard it for a fresh replay. + workflowExecution = { + type: 'retained', + session: workflowResult.session, + }; throw workflowResult.suspension; } @@ -1665,7 +1657,7 @@ export function workflowEntrypoint( workflowRunId: runId, loopIteration, replayMs: Date.now() - replayStart, - executionMode, + executionMode: workflowExecution.type, }); // Workflow completed. Send the snapshot but do NOT @@ -1790,15 +1782,7 @@ export function workflowEntrypoint( requestId, eventLog: suspensionLog, runReadyBarrier, - prepareForRetention: - workflowExecution.type === 'retained', }); - if ( - workflowExecution.type === 'retained' && - !suspensionResult.retainedStepInputsSafe - ) { - workflowExecution = { type: 'replay' }; - } } catch (suspensionError) { // A suspension create whose stale (412) rejection // survived the in-guard reload retries: schedule an @@ -1889,6 +1873,28 @@ export function workflowEntrypoint( return; } eventsCursor = suspensionLog.cursor; + + // Open hooks/waits in the cumulative log (this + // suspension's writes are already merged in), computed + // once for the retention decision here and the + // delta/turbo gates below. + const openHookWaitState = + openHookAndWaitState(cachedEvents); + + // The single retention decision: keep the parked + // session only across a pure step boundary with no + // out-of-band continuation source and provably + // passive step inputs. + if ( + workflowExecution.type === 'retained' && + !( + canRetainWorkflowSession(err, openHookWaitState) && + suspensionResult.retainedStepInputsSafe + ) + ) { + workflowExecution = { type: 'replay' }; + } + preStepBlockingMs += suspensionResult.hookCreationMs; if ( suspensionResult.hasAttributeEvents && @@ -2209,12 +2215,6 @@ export function workflowEntrypoint( // this the replay-budget check at the top of the // next loop iteration would (incorrectly) charge // the step body against the budget. - // Open hooks/waits in the cumulative log, computed - // once for the two gates below. - const openHookWaitState = openHookAndWaitState( - cachedEvents ?? [] - ); - // Inline-delta fast path gate. We request the delta — // and on the next iteration consume it in place of the // events.list — only when ALL hold: diff --git a/packages/core/src/runtime/suspension-handler.ts b/packages/core/src/runtime/suspension-handler.ts index c23ab2e878..72f5411f3c 100644 --- a/packages/core/src/runtime/suspension-handler.ts +++ b/packages/core/src/runtime/suspension-handler.ts @@ -69,11 +69,6 @@ export interface SuspensionHandlerParams { * where `run_started` was already awaited up front. */ runReadyBarrier?: Promise; - /** - * Prepare new step inputs without traversing workflow-owned objects during - * serialization. Enabled only while the caller is holding a retained VM. - */ - prepareForRetention?: boolean; } /** @@ -225,7 +220,6 @@ export async function handleSuspension({ requestId, eventLog, runReadyBarrier, - prepareForRetention = false, }: SuspensionHandlerParams): Promise { const runId = run.runId; @@ -519,20 +513,13 @@ export async function handleSuspension({ // standard built-ins. If any input in the batch is not provably passive, // the caller demotes the session so the side effects land in a VM that is // about to be discarded, exactly like the pre-retention runtime. - let retainedStepInputsSafe = true; - if (prepareForRetention) { - for (const queueItem of stepItems) { - if (!stepsNeedingCreation.has(queueItem.correlationId)) continue; - if ( - queueItem.thisVal !== undefined || - queueItem.closureVars !== undefined || - !queueItem.args.every(isPrimitiveStepArgument) - ) { - retainedStepInputsSafe = false; - break; - } - } - } + const retainedStepInputsSafe = stepItems.every( + (item) => + !stepsNeedingCreation.has(item.correlationId) || + (item.thisVal === undefined && + item.closureVars === undefined && + item.args.every(isPrimitiveStepArgument)) + ); // Lazy inline start: defer the step_created write for up to // `getMaxInlineSteps()` steps the caller will run inline (in parallel). Each diff --git a/packages/core/src/workflow.ts b/packages/core/src/workflow.ts index 4088a0888f..f56580e0d0 100644 --- a/packages/core/src/workflow.ts +++ b/packages/core/src/workflow.ts @@ -1060,10 +1060,13 @@ async function createWorkflowSession({ const failWorkflow = async (error: unknown): Promise => { // Control-flow signals are handled by the runtime and do not mean the - // workflow has terminally failed. `onWorkflowError` already moved the - // state machine (e.g. to `replay` on divergence, so a later resume falls - // back instead of throwing) — leave it alone. + // workflow has terminally failed. `onWorkflowError` usually already moved + // the state machine, but a divergence can also arrive via a step + // promise's direct rejection (bypassing `onWorkflowError`) — demote so + // every control-flow path converges on `replay` and a later resume falls + // back instead of throwing. if (WorkflowSuspension.is(error) || ReplayDivergenceError.is(error)) { + if (state.type === 'running') state = { type: 'replay' }; throw error; } state = { type: 'completed' }; From db1665f3bb0290808e0d2659692c80a278dd4869 Mon Sep 17 00:00:00 2001 From: Nathan Colosimo <110621881+NathanColosimo@users.noreply.github.com> Date: Fri, 24 Jul 2026 16:06:53 -0700 Subject: [PATCH 35/43] mark sandbox API removals as a major change --- .changeset/vm-determinism-hardening.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.changeset/vm-determinism-hardening.md b/.changeset/vm-determinism-hardening.md index 8c3bbdd6a2..e8ea1d47ae 100644 --- a/.changeset/vm-determinism-hardening.md +++ b/.changeset/vm-determinism-hardening.md @@ -1,6 +1,6 @@ --- -'@workflow/core': minor -'workflow': minor +'@workflow/core': major +'workflow': major --- Deterministic sandbox hardening: `crypto.subtle.digest` in workflow functions now computes synchronously via `node:crypto` (byte-identical values, deterministic timing under replay); `Atomics.waitAsync`, async `WebAssembly` compilation, `WeakRef`, and `FinalizationRegistry` are no longer exposed (wall-clock timing and GC observation cannot be replayed). From 8cea5289e1ec9ad8c718dc1d0ee7b027ff9abb5c Mon Sep 17 00:00:00 2001 From: Nathan Colosimo <110621881+NathanColosimo@users.noreply.github.com> Date: Mon, 27 Jul 2026 13:11:21 -0700 Subject: [PATCH 36/43] simplify retention further: one staleness mechanism (generation bump on suspend), whole predicate in canRetainWorkflowSession, lazy hook/wait scan, prewarm on resume path --- packages/core/src/events-consumer.ts | 7 +- packages/core/src/private.ts | 11 +- packages/core/src/runtime.ts | 114 +++++++++++------- .../core/src/runtime/suspension-handler.ts | 4 +- packages/core/src/telemetry.ts | 9 +- packages/core/src/workflow.ts | 44 ++++--- 6 files changed, 115 insertions(+), 74 deletions(-) diff --git a/packages/core/src/events-consumer.ts b/packages/core/src/events-consumer.ts index 7b4afd6a9a..cc05c6f975 100644 --- a/packages/core/src/events-consumer.ts +++ b/packages/core/src/events-consumer.ts @@ -69,7 +69,7 @@ export interface EventsConsumerOptions { export class EventsConsumer { eventIndex: number; - readonly events: Event[] = []; + readonly events: Event[]; readonly callbacks: EventConsumerCallback[] = []; private onConsumedEvent?: (event: Event) => void; private onUnconsumedEvent: (event: Event) => void; @@ -79,6 +79,9 @@ export class EventsConsumer { private unconsumedCheckVersion = 0; constructor(events: Event[], options: EventsConsumerOptions) { + // Own copy: the runtime mutates its event array in place, and a retained + // session must only observe new events through append() so resume()'s + // strict-extension check stays meaningful. this.events = [...events]; this.eventIndex = 0; this.onConsumedEvent = options.onConsumedEvent; @@ -87,7 +90,7 @@ export class EventsConsumer { } append(events: Event[]): void { - this.events.push(...events); + for (const event of events) this.events.push(event); process.nextTick(this.consume); } diff --git a/packages/core/src/private.ts b/packages/core/src/private.ts index 2f7dfd1e72..a40fc0d7ee 100644 --- a/packages/core/src/private.ts +++ b/packages/core/src/private.ts @@ -136,10 +136,13 @@ export interface WorkflowOrchestratorContext { worldCapabilities?: WorldCapabilities; globalThis: typeof globalThis; /** - * Increments on every retained-session resume. Suspension signals capture - * it when scheduled and no-op if it moved — a timer queued at boundary N - * must not signal after the session resumed into boundary N+1, so the - * state machine never has to reason about cross-boundary timer ordering. + * Increments when a suspension is accepted and on every retained-session + * resume. STEP suspension signals capture it when scheduled and no-op if + * it moved (see step.ts) — this drops same-boundary sibling signals and + * timers queued at boundary N that would fire after the session resumed + * into boundary N+1. Sleep/hook/attribute signals are intentionally + * unguarded: their presence makes the boundary unretainable, so a late + * signal correctly demotes the session (workflow.ts `onWorkflowError`). */ suspensionGeneration: number; eventsConsumer: EventsConsumer; diff --git a/packages/core/src/runtime.ts b/packages/core/src/runtime.ts index 13ac06db10..f41083f4e9 100644 --- a/packages/core/src/runtime.ts +++ b/packages/core/src/runtime.ts @@ -287,10 +287,11 @@ function rootRunIdFrom( * `wait_completed`, which the wait timer can resolve with * `wait_completed`). * - * This gates the inline-delta fast path and turbo's forced optimistic start. - * A terminal-step delta can omit an event appended concurrently after that - * write. With no open hook or wait, only cancellation can do so, and observing - * it one replay late is safe because the next entity write is rejected. + * This gates VM retention, the inline-delta fast path, and turbo's forced + * optimistic start. A terminal-step delta can omit an event appended + * concurrently after that write. With no open hook or wait, only cancellation + * can do so, and observing it one replay late is safe because the next entity + * write is rejected. * * Step-body `attr_set` writes are NOT a concern: they land before the * step's terminal write and are therefore already inside the returned @@ -328,11 +329,15 @@ function openHookAndWaitState(events: Event[]): { } /** - * Retain only pure step boundaries (every suspension item is a step — any - * other item type, present or future, is unretainable by default) with no - * out-of-band continuation source: attributes require replay; hooks and - * waits can wake another invocation. `WORKFLOW_RETAINED_VM=0` disables - * retention entirely. + * The whole retention predicate: keep the session only for a pure step + * boundary (every suspension item is a step — any other item type, present + * or future, is unretainable by default) whose new step inputs serialized + * without executing workflow code, with no out-of-band continuation source: + * attributes require replay; hooks and waits can wake another invocation. + * `WORKFLOW_RETAINED_VM=0` disables retention entirely. + * + * The open hook/wait scan is O(events), so it is taken through a lazy getter + * and consulted last, after every cheap check has passed. * * Quiescence assumes workflow code stays inside the sandbox's determinism * contract. Escaping to the host realm (e.g. recovering the host `Function` @@ -342,15 +347,19 @@ function openHookAndWaitState(events: Event[]): { */ function canRetainWorkflowSession( suspension: WorkflowSuspension, - openHookWaitState: { openHook: boolean; openWait: boolean } + stepInputsSafe: boolean, + getOpenHookWaitState: () => ReturnType ): boolean { - return ( - isVmRetentionEnabled() && - suspension.steps.length > 0 && - suspension.steps.every((item) => item.type === 'step') && - !openHookWaitState.openHook && - !openHookWaitState.openWait - ); + if ( + !isVmRetentionEnabled() || + !stepInputsSafe || + suspension.steps.length === 0 || + !suspension.steps.every((item) => item.type === 'step') + ) { + return false; + } + const { openHook, openWait } = getOpenHookWaitState(); + return !openHook && !openWait; } /** @@ -1199,7 +1208,7 @@ export function workflowEntrypoint( } // end if (!workflowRun) // Resolve the encryption key for this run's deployment. - // Used eagerly here since both runWorkflow (input + // Used eagerly here since both executeWorkflow (input // hydration / hook payload decryption) and the run_failed // dehydrate path below need it. Memoized accessor: first // call triggers the actual fetch / HKDF derivation, @@ -1560,6 +1569,15 @@ export function workflowEntrypoint( executionMode: workflowExecution.type, }); replayStart = Date.now(); + // Start every missing decrypt/decompress operation up + // front (already-prepared payloads are skipped). Web + // Crypto work overlaps VM setup on the replay path and + // the appended events' consumption on the resume path; + // consumers still deserialize and resolve in event order. + const payloadPrewarm = replayPayloadCache.prewarm( + workflowRun, + events + ); let workflowResult: WorkflowExecutionResult = workflowExecution.type === 'retained' ? await executeWorkflow({ @@ -1571,14 +1589,6 @@ export function workflowEntrypoint( if (workflowResult.type === 'replay') { workflowExecution = { type: 'replay' }; - // Start every missing decrypt/decompress operation - // before VM setup. Web Crypto work can overlap bundle - // evaluation; consumers still deserialize and resolve - // in event order. - const payloadPrewarm = replayPayloadCache.prewarm( - workflowRun, - events - ); workflowResult = await executeWorkflow({ type: 'replay', workflowCode, @@ -1592,8 +1602,8 @@ export function workflowEntrypoint( runReadyBarrier, worldCapabilities: world.capabilities, }); - await payloadPrewarm; } + await payloadPrewarm; if (workflowResult.type === 'suspended') { // Park the live session; the suspension catch below @@ -1658,7 +1668,7 @@ export function workflowEntrypoint( return; } catch (err) { if (WorkflowSuspension.is(err)) { - // Synchronous `runWorkflow` duration for THIS + // Synchronous `executeWorkflow` duration for THIS // suspension only — anchors the `finalSchedulingReplay` // telemetry field below (see // StepLatencyTracking.replayMs). This is the FINAL @@ -1682,7 +1692,10 @@ export function workflowEntrypoint( // server spans are heavily sampled in production // (~7%), and client spans can't be filtered by SDK // version, so neither can serve as the dashboard's - // exact TTFS decomposition. + // exact TTFS decomposition. On a retained-VM + // resume this measures the resume (typically ~0ms), + // not a replay, so the field's distribution is + // bimodal once retention is active. const replayDurationMs = Date.now() - replayStart; runtimeLogger.debug('Workflow suspended', { workflowRunId: runId, @@ -1829,11 +1842,22 @@ export function workflowEntrypoint( eventsCursor = suspensionLog.cursor; // Open hooks/waits in the cumulative log (this - // suspension's writes are already merged in), computed - // once for the retention decision here and the - // delta/turbo gates below. - const openHookWaitState = - openHookAndWaitState(cachedEvents); + // suspension's writes are already merged in). Computed + // lazily, at most once, and shared between the + // retention decision here and the delta/turbo gates + // below — the attr-detour and hook-conflict paths + // return/continue before the gates and usually + // short-circuit before ever scanning the log. + // Narrowed alias: the closure below would otherwise + // lose `cachedEvents`'s non-null narrowing. Nothing in + // this catch scope reassigns the array. + const suspensionEvents = cachedEvents; + let openHookWaitMemo: + | ReturnType + | undefined; + const getOpenHookWaitState = () => + (openHookWaitMemo ??= + openHookAndWaitState(suspensionEvents)); // The single retention decision: keep the parked // session only across a pure step boundary with no @@ -1841,9 +1865,10 @@ export function workflowEntrypoint( // passive step inputs. if ( workflowExecution.type === 'retained' && - !( - canRetainWorkflowSession(err, openHookWaitState) && - suspensionResult.retainedStepInputsSafe + !canRetainWorkflowSession( + err, + suspensionResult.retainedStepInputsSafe, + getOpenHookWaitState ) ) { workflowExecution = { type: 'replay' }; @@ -2166,13 +2191,10 @@ export function workflowEntrypoint( return; } - // Execute inline step. Pause the replay budget - // for the duration of the step body — step - // duration is bounded by the platform's function - // maxDuration, not by the replay timeout. Without - // this the replay-budget check at the top of the - // next loop iteration would (incorrectly) charge - // the step body against the budget. + // Open hooks/waits are consulted by all three gates + // below; resolve the memoized scan once here. + const openHookWaitState = getOpenHookWaitState(); + // Inline-delta fast path gate. We request the delta — // and on the next iteration consume it in place of the // events.list — only when ALL hold: @@ -2338,7 +2360,7 @@ export function workflowEntrypoint( // snapshot has a local-clock createdAt, so under // turbo only the run-id ULID timestamp is trusted. const latencyTracking = computeStepLatencyTracking({ - events: cachedEvents ?? [], + events: cachedEvents, invocationStartedClean: invocationStartedClean === true, runCreatedAtMs: @@ -2369,7 +2391,7 @@ export function workflowEntrypoint( // this is a no-op outside guarded deployments; Worlds // that don't enforce the guard ignore it. const inlineClaimStateUpdatedAt = - stateUpdatedAtForCreate(cachedEvents ?? []); + stateUpdatedAtForCreate(cachedEvents); replayBudget.pause(); let stepResults: Awaited< diff --git a/packages/core/src/runtime/suspension-handler.ts b/packages/core/src/runtime/suspension-handler.ts index 6b18d1d233..7468acb79b 100644 --- a/packages/core/src/runtime/suspension-handler.ts +++ b/packages/core/src/runtime/suspension-handler.ts @@ -34,7 +34,9 @@ import { type MutableEventLog, withPreconditionRetry } from './helpers.js'; // Serializing a primitive executes no code of any kind. BigInt is excluded: // its encoding calls a prototype method. Widened to plain data and standard -// built-ins by the retained-input walker in a follow-up. +// built-ins by the retained-input walker in a follow-up. (Distinct from +// replay-payload-cache's `isMemoizablePrimitive`, a size-gated memoization +// filter — do not merge them.) function isPrimitiveStepArgument(value: unknown): boolean { return ( value === null || diff --git a/packages/core/src/telemetry.ts b/packages/core/src/telemetry.ts index 0c3a32f9d5..fd126abb3e 100644 --- a/packages/core/src/telemetry.ts +++ b/packages/core/src/telemetry.ts @@ -245,13 +245,14 @@ export async function trace( span.setStatus({ code: otel.SpanStatusCode.OK }); return result; } catch (e) { - span.setStatus({ - code: otel.SpanStatusCode.ERROR, - message: (e as Error).message, - }); if (WorkflowSuspension.is(e)) { span.setStatus({ code: otel.SpanStatusCode.OK }); applyWorkflowSuspensionToSpan(e, span); + } else { + span.setStatus({ + code: otel.SpanStatusCode.ERROR, + message: (e as Error).message, + }); } throw e; } finally { diff --git a/packages/core/src/workflow.ts b/packages/core/src/workflow.ts index 2c0b214035..d86749ef67 100644 --- a/packages/core/src/workflow.ts +++ b/packages/core/src/workflow.ts @@ -215,6 +215,8 @@ export async function executeWorkflow( execution = resumed.execution; break; } + // No default: exhaustiveness is enforced by definite assignment of + // `session`/`execution` above. } span?.setAttributes({ @@ -237,6 +239,12 @@ export async function executeWorkflow( }); } +/** + * Single-shot replay: execute the workflow over `events` and either return + * its output or throw its suspension. Kept for the extensive existing test + * suites — production code goes through `executeWorkflow`, which can also + * resume a retained session. + */ export async function runWorkflow( workflowCode: string, workflowRun: WorkflowRun, @@ -334,23 +342,20 @@ async function createWorkflowSession({ state = WorkflowSuspension.is(error) ? { type: 'suspended', suspension: error } : { type: 'replay' }; + // Each parked step consumer schedules its own (identical) suspension + // signal; the first one lands here, and bumping the generation makes + // the step-consumer guard drop the rest at fire time. + workflowContext.suspensionGeneration++; interruption.reject(error); return; } - case 'suspended': { - // A suspended VM is quiescent, but each parked step consumer - // signals its own (identical) suspension via scheduleWhenIdle — - // absorb those duplicates, treat anything else as unretainable. - // The suspension counts are all derived from `steps`, so identical - // items mean an identical boundary. - const { steps } = state.suspension; - const isDuplicateSignal = - WorkflowSuspension.is(error) && - error.steps.length === steps.length && - error.steps.every((item, index) => item === steps[index]); - if (!isDuplicateSignal) state = { type: 'replay' }; + case 'suspended': + // Same-boundary duplicates were staled by the generation bump above, + // so anything landing here is out-of-band — an unguarded sleep/hook/ + // attribute signal or a divergence. Those boundaries are unretainable + // (the runtime demotes them too), so fall back to replay. + state = { type: 'replay' }; return; - } case 'replay': case 'completed': return; @@ -1070,10 +1075,15 @@ async function createWorkflowSession({ resume(nextEvents) { switch (state.type) { case 'suspended': { - const consumedEvents = eventsConsumer.events; + // The full O(known events) prefix compare is required: the runtime + // usually grows one array in place, but its wait-completion and + // stale-reload paths REPLACE the array with a fresh full fetch, so + // a rewritten prefix is a real input, not paranoia. The compares + // are cheap string equality and dwarfed by the replay they avoid. + const knownEvents = eventsConsumer.events; const isStrictExtension = - nextEvents.length > consumedEvents.length && - consumedEvents.every( + nextEvents.length > knownEvents.length && + knownEvents.every( (event, index) => event.eventId === nextEvents[index]?.eventId ); if (!isStrictExtension) { @@ -1083,7 +1093,7 @@ async function createWorkflowSession({ const interruption = withResolvers(); state = { type: 'running', interruption }; workflowContext.suspensionGeneration++; - eventsConsumer.append(nextEvents.slice(consumedEvents.length)); + eventsConsumer.append(nextEvents.slice(knownEvents.length)); return { type: 'resumed', execution: waitForExecution(interruption), From a16acd34e43696047ce9ef542685b23bf83cb959 Mon Sep 17 00:00:00 2001 From: Nathan Colosimo <110621881+NathanColosimo@users.noreply.github.com> Date: Mon, 27 Jul 2026 20:33:42 -0700 Subject: [PATCH 37/43] simplify session API and tests: replace executeWorkflow overloads with replayWorkflow/resumeWorkflow, drop low-value events-consumer tests, compact session and retained-loop tests --- packages/core/src/events-consumer.test.ts | 27 -- packages/core/src/retained-vm-loop.test.ts | 107 ++---- packages/core/src/runtime.ts | 19 +- .../src/workflow-session-telemetry.test.ts | 5 +- packages/core/src/workflow.test.ts | 344 +++++++----------- packages/core/src/workflow.ts | 144 ++++---- 6 files changed, 255 insertions(+), 391 deletions(-) diff --git a/packages/core/src/events-consumer.test.ts b/packages/core/src/events-consumer.test.ts index 3e320b16c7..ecf828f728 100644 --- a/packages/core/src/events-consumer.test.ts +++ b/packages/core/src/events-consumer.test.ts @@ -45,15 +45,6 @@ describe('EventsConsumer', () => { expect(consumer.eventIndex).toBe(0); expect(consumer.callbacks).toEqual([]); }); - - it('should own its event array', () => { - const events = [createMockEvent()]; - const consumer = new EventsConsumer(events, defaultOptions); - - events.push(createMockEvent({ id: 'event-2' })); - - expect(consumer.events).toHaveLength(1); - }); }); describe('subscribe', () => { @@ -91,24 +82,6 @@ describe('EventsConsumer', () => { expect(callback).toHaveBeenCalledWith(event); expect(callback).toHaveBeenCalledTimes(1); }); - - it('should consume appended events', async () => { - const event = createMockEvent(); - const consumer = new EventsConsumer([], defaultOptions); - const callback = vi - .fn() - .mockImplementation((value: Event | null) => - value ? EventConsumerResult.Finished : EventConsumerResult.NotConsumed - ); - consumer.subscribe(callback); - await waitForNextTick(); - - consumer.append([event]); - await waitForNextTick(); - - expect(callback).toHaveBeenLastCalledWith(event); - expect(consumer.eventIndex).toBe(1); - }); }); describe('consume (implicit)', () => { diff --git a/packages/core/src/retained-vm-loop.test.ts b/packages/core/src/retained-vm-loop.test.ts index d24b59bcc2..3f7ed086c0 100644 --- a/packages/core/src/retained-vm-loop.test.ts +++ b/packages/core/src/retained-vm-loop.test.ts @@ -201,12 +201,15 @@ async function drive(runId: string, workflowCode = twoStepWorkflow) { await workflowEntrypoint(workflowCode)(new Request('https://example.test')); - const runCompleted = createdEvents.find( - (e) => e.eventType === 'run_completed' - ); + const output = createdEvents.find((e) => e.eventType === 'run_completed') + ?.eventData?.output as Uint8Array | undefined; return { vmBuilds: createContextSpy.mock.calls.length, - output: runCompleted?.eventData?.output as Uint8Array | undefined, + output, + result: + output === undefined + ? undefined + : await hydrateWorkflowReturnValue(output, runId, undefined, []), }; } @@ -222,109 +225,59 @@ describe('retained VM through the inline replay loop', () => { it('rebuilds the VM once per replay under the kill switch (WORKFLOW_RETAINED_VM=0)', async () => { process.env.WORKFLOW_RETAINED_VM = '0'; - const { vmBuilds, output } = await drive('wrun_retained_off'); - expect(output).toBeInstanceOf(Uint8Array); - // A 2-step sequential workflow suspends twice then completes → >1 replay, - // each building a fresh VM. + const { vmBuilds, result } = await drive('wrun_retained_off'); + expect(result).toBe(30); + // 2 sequential steps → 2 suspensions + completion → 3 replays, each + // building a fresh VM. expect(vmBuilds).toBeGreaterThan(1); }); - it('builds the VM once and resumes it when retention is ON (the default)', async () => { - // Baseline via the kill switch, to compare the dehydrated result against. + it('builds the VM once when retention is ON (the default), with byte-identical output', async () => { + // Baseline via the kill switch, to compare the dehydrated bytes against. process.env.WORKFLOW_RETAINED_VM = '0'; const off = await drive('wrun_retained_baseline_off'); createContextSpy.mockClear(); - - // Unset → retention is ON by default. delete process.env.WORKFLOW_RETAINED_VM; - const on = await drive('wrun_retained_on'); - expect(on.output).toBeInstanceOf(Uint8Array); - // The whole run is served by a single VM: built once on the first pass, - // resumed (not rebuilt) for every subsequent step. + const on = await drive('wrun_retained_on'); + expect(on.result).toBe(30); + // One VM for the whole run: built on the first pass, resumed after. expect(on.vmBuilds).toBe(1); - // And it produces the identical dehydrated result as the replay path. expect(on.output).toEqual(off.output); }); - it('matches cold replay when argument serialization mutates workflow state', async () => { + it.each([ + ['an argument getter', impureArgsWorkflow, 'impure_args'], + ['a custom serializer', impureSerializerWorkflow, 'impure_serializer'], + ])('matches cold replay when %s mutates workflow state', async (_name, workflowCode, slug) => { process.env.WORKFLOW_RETAINED_VM = '0'; - const off = await drive( - 'wrun_retained_impure_args_off', - impureArgsWorkflow - ); + const off = await drive(`wrun_${slug}_off`, workflowCode); createContextSpy.mockClear(); - delete process.env.WORKFLOW_RETAINED_VM; - const on = await drive('wrun_retained_impure_args_on', impureArgsWorkflow); - - expect(on.vmBuilds).toBeGreaterThan(1); - expect( - await hydrateWorkflowReturnValue( - off.output, - 'wrun_retained_impure_args_off', - undefined, - [] - ) - ).toBe(0); - expect( - await hydrateWorkflowReturnValue( - on.output, - 'wrun_retained_impure_args_on', - undefined, - [] - ) - ).toBe(0); - }); - - it('matches cold replay when a custom serializer mutates workflow state', async () => { - process.env.WORKFLOW_RETAINED_VM = '0'; - const off = await drive( - 'wrun_retained_impure_serializer_off', - impureSerializerWorkflow - ); - createContextSpy.mockClear(); - - delete process.env.WORKFLOW_RETAINED_VM; - const on = await drive( - 'wrun_retained_impure_serializer_on', - impureSerializerWorkflow - ); + const on = await drive(`wrun_${slug}_on`, workflowCode); + // The boundary demoted (multiple VMs), and the result is what a cold + // replay computes: the serialization-time mutation is NOT visible. expect(on.vmBuilds).toBeGreaterThan(1); - expect( - await hydrateWorkflowReturnValue( - off.output, - 'wrun_retained_impure_serializer_off', - undefined, - [] - ) - ).toBe(0); - expect( - await hydrateWorkflowReturnValue( - on.output, - 'wrun_retained_impure_serializer_on', - undefined, - [] - ) - ).toBe(0); + expect(off.result).toBe(0); + expect(on.result).toBe(0); }); it('demotes retention when any input in a parallel batch is unsafe', async () => { - const { vmBuilds, output } = await drive( + const { vmBuilds, result } = await drive( 'wrun_retained_mixed_batch', mixedBatchWorkflow ); - expect(output).toBeInstanceOf(Uint8Array); + expect(result).toBe(30); expect(vmBuilds).toBeGreaterThan(1); }); it('retains a VM that used the synchronous crypto.subtle.digest', async () => { - const { vmBuilds, output } = await drive( + const { vmBuilds, result } = await drive( 'wrun_retained_digest', digestWorkflow ); - expect(output).toBeInstanceOf(Uint8Array); + expect(result).toBe(30); expect(vmBuilds).toBe(1); }); }); diff --git a/packages/core/src/runtime.ts b/packages/core/src/runtime.ts index f41083f4e9..ab6948d1b9 100644 --- a/packages/core/src/runtime.ts +++ b/packages/core/src/runtime.ts @@ -101,7 +101,8 @@ import { import { getErrorName, getErrorStack, normalizeUnknownError } from './types.js'; import { buildWorkflowSuspensionMessage } from './util.js'; import { - executeWorkflow, + replayWorkflow, + resumeWorkflow, type WorkflowExecutionResult, type WorkflowSession, } from './workflow.js'; @@ -1208,7 +1209,7 @@ export function workflowEntrypoint( } // end if (!workflowRun) // Resolve the encryption key for this run's deployment. - // Used eagerly here since both executeWorkflow (input + // Used eagerly here since both workflow execution (input // hydration / hook payload decryption) and the run_failed // dehydrate path below need it. Memoized accessor: first // call triggers the actual fetch / HKDF derivation, @@ -1580,17 +1581,15 @@ export function workflowEntrypoint( ); let workflowResult: WorkflowExecutionResult = workflowExecution.type === 'retained' - ? await executeWorkflow({ - type: 'resume', - session: workflowExecution.session, - events, - }) + ? await resumeWorkflow( + workflowExecution.session, + events + ) : { type: 'replay' }; if (workflowResult.type === 'replay') { workflowExecution = { type: 'replay' }; - workflowResult = await executeWorkflow({ - type: 'replay', + workflowResult = await replayWorkflow({ workflowCode, workflowRun, events, @@ -1668,7 +1667,7 @@ export function workflowEntrypoint( return; } catch (err) { if (WorkflowSuspension.is(err)) { - // Synchronous `executeWorkflow` duration for THIS + // Synchronous workflow-execution duration for THIS // suspension only — anchors the `finalSchedulingReplay` // telemetry field below (see // StepLatencyTracking.replayMs). This is the FINAL diff --git a/packages/core/src/workflow-session-telemetry.test.ts b/packages/core/src/workflow-session-telemetry.test.ts index 49058a4585..f2305c794c 100644 --- a/packages/core/src/workflow-session-telemetry.test.ts +++ b/packages/core/src/workflow-session-telemetry.test.ts @@ -8,7 +8,7 @@ import type { WorkflowRun } from '@workflow/world'; import { afterAll, afterEach, beforeAll, describe, expect, it } from 'vitest'; import { ReplayPayloadCache } from './replay-payload-cache.js'; import { dehydrateWorkflowArguments } from './serialization.js'; -import { executeWorkflow } from './workflow.js'; +import { replayWorkflow } from './workflow.js'; const exporter = new InMemorySpanExporter(); const provider = new BasicTracerProvider(); @@ -45,8 +45,7 @@ describe('retained workflow telemetry', () => { async function workflow() { await step(); } globalThis.__private_workflows = new Map([["workflow", workflow]]);`; - const result = await executeWorkflow({ - type: 'replay', + const result = await replayWorkflow({ workflowCode, workflowRun, events: [], diff --git a/packages/core/src/workflow.test.ts b/packages/core/src/workflow.test.ts index 6830603819..93166df6f8 100644 --- a/packages/core/src/workflow.test.ts +++ b/packages/core/src/workflow.test.ts @@ -14,7 +14,7 @@ import { hydrateWorkflowReturnValue, } from './serialization.js'; import { createContext } from './vm/index.js'; -import { executeWorkflow, runWorkflow } from './workflow.js'; +import { replayWorkflow, resumeWorkflow, runWorkflow } from './workflow.js'; // No encryption key = encryption disabled const noEncryptionKey = undefined; @@ -222,249 +222,181 @@ describe('runWorkflow', () => { ).toEqual(3); }); - it('should retain workflow execution across sequential step suspensions', async () => { - const ops: Promise[] = []; - const workflowRun: WorkflowRun = { - runId: 'wrun_retained', + describe('workflow sessions (retained VM)', () => { + const sessionRun = async (runId: string): Promise => ({ + runId, workflowName: 'workflow', status: 'running', - input: await dehydrateWorkflowArguments( - [], - 'wrun_retained', - noEncryptionKey, - ops - ), + input: await dehydrateWorkflowArguments([], runId, noEncryptionKey, []), createdAt: new Date('2024-01-01T00:00:00.000Z'), updatedAt: new Date('2024-01-01T00:00:00.000Z'), startedAt: new Date('2024-01-01T00:00:00.000Z'), deploymentId: 'test-deployment', - }; - const workflowCode = ` - const add = globalThis[Symbol.for("WORKFLOW_USE_STEP")]("add"); - async function workflow() { - console.log("retained:entered"); - const first = await add(1, 2); - console.log("retained:continued"); - const second = await add(first, 3); - return second; - } - ${getWorkflowTransformCode('workflow')}`; - const log = vi.spyOn(console, 'log').mockImplementation(() => undefined); - const events: Event[] = []; - let eventNumber = 0; - const appendStepEvents = async ( - correlationId: string, + }); + + const replay = ( + workflowRun: WorkflowRun, + workflowCode: string, + events: Event[] + ) => + replayWorkflow({ + workflowCode, + workflowRun, + events, + encryptionKey: noEncryptionKey, + replayPayloadCache: new ReplayPayloadCache(noEncryptionKey), + }); + + // Append the step_created/started/completed triplet for the suspension's + // single pending step, as the runtime's inline execution would. + let eventCounter = 0; + const completeStep = async ( + run: WorkflowRun, + events: Event[], + suspension: WorkflowSuspension, result: number ): Promise => { - const createdAt = new Date(`2024-01-01T00:00:0${eventNumber + 1}.000Z`); + const step = suspension.steps[0]; + assert(step?.type === 'step'); + const base = { + runId: run.runId, + correlationId: step.correlationId, + createdAt: new Date(Date.UTC(2024, 0, 1, 0, 0, ++eventCounter)), + }; events.push( { - eventId: `event-${eventNumber++}`, - runId: workflowRun.runId, + ...base, + eventId: `event-${eventCounter}-created`, eventType: 'step_created', - correlationId, eventData: { stepName: 'add' }, - createdAt, - }, + } as Event, { - eventId: `event-${eventNumber++}`, - runId: workflowRun.runId, + ...base, + eventId: `event-${eventCounter}-started`, eventType: 'step_started', - correlationId, eventData: { stepName: 'add' }, - createdAt, - }, + } as Event, { - eventId: `event-${eventNumber++}`, - runId: workflowRun.runId, + ...base, + eventId: `event-${eventCounter}-completed`, eventType: 'step_completed', - correlationId, eventData: { stepName: 'add', result: await dehydrateStepReturnValue( result, - workflowRun.runId, + run.runId, noEncryptionKey, - ops + [] ), }, - createdAt, - } + } as Event ); }; - const first = await executeWorkflow({ - type: 'replay', - workflowCode, - workflowRun, - events, - encryptionKey: noEncryptionKey, - replayPayloadCache: new ReplayPayloadCache(noEncryptionKey), - }); - assert(first.type === 'suspended'); - const firstStep = first.suspension.steps[0]; - assert(firstStep?.type === 'step'); - - await appendStepEvents(firstStep.correlationId, 3); + it('resumes one VM across sequential step boundaries', async () => { + const run = await sessionRun('wrun_retained'); + const workflowCode = ` + const add = globalThis[Symbol.for("WORKFLOW_USE_STEP")]("add"); + async function workflow() { + console.log("retained:entered"); + const first = await add(1, 2); + console.log("retained:continued"); + return await add(first, 3); + } + ${getWorkflowTransformCode('workflow')}`; + const log = vi.spyOn(console, 'log').mockImplementation(() => undefined); + const events: Event[] = []; - const second = await executeWorkflow({ - type: 'resume', - session: first.session, - events, - }); - assert(second.type === 'suspended'); - expect(second.session).toBe(first.session); - const secondStep = second.suspension.steps[0]; - assert(secondStep?.type === 'step'); + const first = await replay(run, workflowCode, events); + assert(first.type === 'suspended'); + await completeStep(run, events, first.suspension, 3); - await appendStepEvents(secondStep.correlationId, 6); + const second = await resumeWorkflow(first.session, events); + assert(second.type === 'suspended'); + expect(second.session).toBe(first.session); + await completeStep(run, events, second.suspension, 6); - const completed = await executeWorkflow({ - type: 'resume', - session: second.session, - events, + const completed = await resumeWorkflow(second.session, events); + assert(completed.type === 'completed'); + expect( + await hydrateWorkflowReturnValue( + completed.output as any, + run.runId, + noEncryptionKey, + [] + ) + ).toBe(6); + // The body ran straight through exactly once — never re-entered. + expect( + log.mock.calls + .map(([message]) => message) + .filter((message) => String(message).startsWith('retained:')) + ).toEqual(['retained:entered', 'retained:continued']); + log.mockRestore(); }); - assert(completed.type === 'completed'); - expect( - await hydrateWorkflowReturnValue( - completed.output as any, - workflowRun.runId, - noEncryptionKey, - ops - ) - ).toBe(6); - expect( - log.mock.calls - .map(([message]) => message) - .filter((message) => String(message).startsWith('retained:')) - ).toEqual(['retained:entered', 'retained:continued']); - log.mockRestore(); - }); + it('declines resume permanently when the event prefix diverges', async () => { + const run = await sessionRun('wrun_retained_divergence'); + const workflowCode = ` + const step = globalThis[Symbol.for("WORKFLOW_USE_STEP")]("step"); + async function workflow() { await step(); } + ${getWorkflowTransformCode('workflow')}`; + const events: Event[] = [ + { + eventId: 'event-run-created', + runId: run.runId, + eventType: 'run_created', + createdAt: new Date('2024-01-01T00:00:00.000Z'), + } as Event, + ]; - it('falls back to replay permanently when the event prefix diverges', async () => { - const ops: Promise[] = []; - const workflowRun: WorkflowRun = { - runId: 'wrun_retained_divergence', - workflowName: 'workflow', - status: 'running', - input: await dehydrateWorkflowArguments( - [], - 'wrun_retained_divergence', - noEncryptionKey, - ops - ), - createdAt: new Date('2024-01-01T00:00:00.000Z'), - updatedAt: new Date('2024-01-01T00:00:00.000Z'), - startedAt: new Date('2024-01-01T00:00:00.000Z'), - deploymentId: 'test-deployment', - }; - const workflowCode = ` - const step = globalThis[Symbol.for("WORKFLOW_USE_STEP")]("step"); - async function workflow() { await step(); } - ${getWorkflowTransformCode('workflow')}`; - const events: Event[] = [ - { - eventId: 'event-run-created', - runId: workflowRun.runId, - eventType: 'run_created', - createdAt: new Date('2024-01-01T00:00:00.000Z'), - }, - ]; + const suspended = await replay(run, workflowCode, events); + assert(suspended.type === 'suspended'); - const suspended = await executeWorkflow({ - type: 'replay', - workflowCode, - workflowRun, - events, - encryptionKey: noEncryptionKey, - replayPayloadCache: new ReplayPayloadCache(noEncryptionKey), - }); - assert(suspended.type === 'suspended'); + // A log whose consumed prefix was rewritten is not a strict extension. + const rewritten = [ + { ...events[0], eventId: 'rewritten-event' } as Event, + { ...events[0], eventId: 'new-event' } as Event, + ]; + expect(await resumeWorkflow(suspended.session, rewritten)).toEqual({ + type: 'replay', + }); - // A log whose consumed prefix was rewritten is not a strict extension. - const rewritten = [ - { ...events[0], eventId: 'rewritten-event' } as Event, - { ...events[0], eventId: 'new-event' } as Event, - ]; - expect( - await executeWorkflow({ - type: 'resume', - session: suspended.session, - events: rewritten, - }) - ).toEqual({ type: 'replay' }); - - // The fallback is permanent, even for a well-formed extension. - expect( - await executeWorkflow({ - type: 'resume', - session: suspended.session, - events, - }) - ).toEqual({ type: 'replay' }); - }); + // The fallback is permanent, even for a well-formed extension. + expect(await resumeWorkflow(suspended.session, events)).toEqual({ + type: 'replay', + }); + }); - it('falls back to replay (not failure) after mid-execution divergence', async () => { - const ops: Promise[] = []; - const workflowRun: WorkflowRun = { - runId: 'wrun_retained_mid_divergence', - workflowName: 'workflow', - status: 'running', - input: await dehydrateWorkflowArguments( - [], - 'wrun_retained_mid_divergence', - noEncryptionKey, - ops - ), - createdAt: new Date('2024-01-01T00:00:00.000Z'), - updatedAt: new Date('2024-01-01T00:00:00.000Z'), - startedAt: new Date('2024-01-01T00:00:00.000Z'), - deploymentId: 'test-deployment', - }; - const workflowCode = ` - const step = globalThis[Symbol.for("WORKFLOW_USE_STEP")]("step"); - async function workflow() { await step(); } - ${getWorkflowTransformCode('workflow')}`; - const events: Event[] = []; + it('demotes to replay (not failure) after mid-execution divergence', async () => { + const run = await sessionRun('wrun_retained_mid_divergence'); + const workflowCode = ` + const step = globalThis[Symbol.for("WORKFLOW_USE_STEP")]("step"); + async function workflow() { await step(); } + ${getWorkflowTransformCode('workflow')}`; + + const suspended = await replay(run, workflowCode, []); + assert(suspended.type === 'suspended'); + + // A strict extension whose appended suffix the VM cannot consume: the + // resume starts, then diverges mid-execution. + const alien = { + eventId: 'event-alien', + runId: run.runId, + eventType: 'hook_received', + correlationId: 'hook_unknown', + eventData: {}, + createdAt: new Date('2024-01-01T00:00:01.000Z'), + } as Event; + await expect(resumeWorkflow(suspended.session, [alien])).rejects.toThrow( + /could not consume event/ + ); - const suspended = await executeWorkflow({ - type: 'replay', - workflowCode, - workflowRun, - events, - encryptionKey: noEncryptionKey, - replayPayloadCache: new ReplayPayloadCache(noEncryptionKey), + // The session demotes to replay — resume() must not throw. + expect(await resumeWorkflow(suspended.session, [])).toEqual({ + type: 'replay', + }); }); - assert(suspended.type === 'suspended'); - - // A strict extension whose appended suffix the VM cannot consume: the - // resume itself starts, then diverges mid-execution. - await expect( - executeWorkflow({ - type: 'resume', - session: suspended.session, - events: [ - { - eventId: 'event-alien', - runId: workflowRun.runId, - eventType: 'hook_received', - correlationId: 'hook_unknown', - eventData: {}, - createdAt: new Date('2024-01-01T00:00:01.000Z'), - } as Event, - ], - }) - ).rejects.toThrow(/could not consume event/); - - // The session must demote to replay — not throw "cannot resume". - expect( - await executeWorkflow({ - type: 'resume', - session: suspended.session, - events: [], - }) - ).toEqual({ type: 'replay' }); }); it('regenerates step correlation IDs independent of startedAt (turbo replay-stability)', async () => { diff --git a/packages/core/src/workflow.ts b/packages/core/src/workflow.ts index d86749ef67..4e44628a60 100644 --- a/packages/core/src/workflow.ts +++ b/packages/core/src/workflow.ts @@ -1,3 +1,4 @@ +import type { Span } from '@opentelemetry/api'; import { ERROR_SLUGS, ReplayDivergenceError, @@ -153,15 +154,8 @@ interface WorkflowSessionOptions { readonly worldCapabilities?: WorldCapabilities; } -type WorkflowExecutionRequest = - | ({ readonly type: 'replay' } & WorkflowSessionOptions) - | { - readonly type: 'resume'; - readonly session: WorkflowSession; - readonly events: Event[]; - }; - -type WorkflowReplayResult = +/** A finished execution attempt: the workflow's output or a live boundary. */ +type WorkflowOutcome = | { readonly type: 'completed'; readonly output: unknown } | { readonly type: 'suspended'; @@ -169,81 +163,96 @@ type WorkflowReplayResult = readonly session: WorkflowSession; }; +/** `resumeWorkflow`'s result: `replay` means "cold-replay me instead". */ export type WorkflowExecutionResult = - | { readonly type: 'replay' } - | WorkflowReplayResult; - -// A fresh replay always completes or suspends; only resuming a session can -// request another replay (stale session or diverged event prefix). -export async function executeWorkflow( - request: { readonly type: 'replay' } & WorkflowSessionOptions -): Promise; -export async function executeWorkflow( - request: WorkflowExecutionRequest -): Promise; -export async function executeWorkflow( - request: WorkflowExecutionRequest + | WorkflowOutcome + | { readonly type: 'replay' }; + +/** + * Cold-start: build a fresh VM session and replay it over `events`. A fresh + * replay always completes or suspends. + */ +export function replayWorkflow( + options: WorkflowSessionOptions +): Promise { + return traceExecution( + 'replay', + options.workflowRun, + options.events, + async (span) => { + const { session, execution } = await createWorkflowSession(options); + return awaitOutcome(session, execution, span); + } + ); +} + +/** + * Warm-start: advance a retained session by appending events. Returns + * `{ type: 'replay' }` when the session declines (demoted, or `events` is + * not a strict extension of what it already consumed). + */ +export function resumeWorkflow( + session: WorkflowSession, + events: Event[] ): Promise { - const workflowRun = - request.type === 'replay' - ? request.workflowRun - : request.session.workflowRun; - const mode = request.type === 'replay' ? 'replay' : 'retained'; + return traceExecution( + 'retained', + session.workflowRun, + events, + async (span) => { + const resumed = session.resume(events); + if (resumed.type === 'replay') return resumed; + return awaitOutcome(session, resumed.execution, span); + } + ); +} - return trace(`workflow.run ${workflowRun.workflowName}`, async (span) => { +function traceExecution( + mode: 'replay' | 'retained', + workflowRun: WorkflowRun, + events: Event[], + fn: (span: Span | undefined) => Promise +): Promise { + return trace(`workflow.run ${workflowRun.workflowName}`, (span) => { span?.setAttributes({ ...Attribute.WorkflowName(workflowRun.workflowName), ...Attribute.WorkflowRunId(workflowRun.runId), ...Attribute.WorkflowRunStatus(workflowRun.status), - ...Attribute.WorkflowEventsCount(request.events.length), + ...Attribute.WorkflowEventsCount(events.length), ...Attribute.WorkflowExecutionMode(mode), }); + return fn(span); + }); +} - let session: WorkflowSession; - let execution: Promise; - switch (request.type) { - case 'replay': { - const started = await createWorkflowSession(request); - session = started.session; - execution = started.execution; - break; - } - case 'resume': { - session = request.session; - const resumed = session.resume(request.events); - if (resumed.type === 'replay') return resumed; - execution = resumed.execution; - break; - } - // No default: exhaustiveness is enforced by definite assignment of - // `session`/`execution` above. - } - +/** Await the execution attempt, converting a suspension into an outcome. */ +async function awaitOutcome( + session: WorkflowSession, + execution: Promise, + span: Span | undefined +): Promise { + span?.setAttributes({ + ...Attribute.WorkflowArgumentsCount(session.argumentCount), + }); + try { + const completed = await execution; span?.setAttributes({ - ...Attribute.WorkflowArgumentsCount(session.argumentCount), + ...Attribute.WorkflowResultType(completed.resultType), }); - - try { - const completed = await execution; - span?.setAttributes({ - ...Attribute.WorkflowResultType(completed.resultType), - }); - return { type: 'completed', output: completed.output }; - } catch (error) { - if (WorkflowSuspension.is(error)) { - if (span) applyWorkflowSuspensionToSpan(error, span); - return { type: 'suspended', suspension: error, session }; - } - throw error; + return { type: 'completed', output: completed.output }; + } catch (error) { + if (WorkflowSuspension.is(error)) { + if (span) applyWorkflowSuspensionToSpan(error, span); + return { type: 'suspended', suspension: error, session }; } - }); + throw error; + } } /** * Single-shot replay: execute the workflow over `events` and either return * its output or throw its suspension. Kept for the extensive existing test - * suites — production code goes through `executeWorkflow`, which can also - * resume a retained session. + * suites — production code goes through `replayWorkflow`/`resumeWorkflow`. */ export async function runWorkflow( workflowCode: string, @@ -270,8 +279,7 @@ export async function runWorkflow( */ worldCapabilities?: WorldCapabilities ): Promise { - const result = await executeWorkflow({ - type: 'replay', + const result = await replayWorkflow({ workflowCode, workflowRun, events, From d9e3b71af190e89a123312dd138e798d9805941a Mon Sep 17 00:00:00 2001 From: Nathan Colosimo <110621881+NathanColosimo@users.noreply.github.com> Date: Tue, 28 Jul 2026 17:03:54 -0700 Subject: [PATCH 38/43] add parallel-batch retention test (sibling signal absorption) and document the unguarded-signaler invariant --- packages/core/src/retained-vm-loop.test.ts | 21 +++++++++++++++++++++ packages/core/src/runtime.ts | 7 +++++++ 2 files changed, 28 insertions(+) diff --git a/packages/core/src/retained-vm-loop.test.ts b/packages/core/src/retained-vm-loop.test.ts index 3f7ed086c0..94a9cd4e8d 100644 --- a/packages/core/src/retained-vm-loop.test.ts +++ b/packages/core/src/retained-vm-loop.test.ts @@ -87,6 +87,18 @@ registerSerializationClass( RetainedSerializerValue ); +// A parallel all-primitive batch: both parked step consumers schedule their +// own (identical) suspension signal for the same boundary — the first one is +// the suspension, the sibling must be absorbed by the generation guard +// without demoting the session. +const parallelBatchWorkflow = `const s1 = globalThis[Symbol.for("WORKFLOW_USE_STEP")]("r_s1"); + const s2 = globalThis[Symbol.for("WORKFLOW_USE_STEP")]("r_s2"); + async function workflow() { + const [a, b] = await Promise.all([s1(1), s2(2)]); + return a + b; + } + globalThis.__private_workflows = new Map([["workflow", workflow]]);`; + // A parallel batch where one sibling's input is unsafe must serialize the // WHOLE batch through the ordinary VM path (all-or-nothing) and demote. const mixedBatchWorkflow = `const s1 = globalThis[Symbol.for("WORKFLOW_USE_STEP")]("r_s1"); @@ -263,6 +275,15 @@ describe('retained VM through the inline replay loop', () => { expect(on.result).toBe(0); }); + it('retains one VM for a parallel batch (sibling suspension signals absorbed)', async () => { + const { vmBuilds, result } = await drive( + 'wrun_retained_parallel_batch', + parallelBatchWorkflow + ); + expect(result).toBe(30); + expect(vmBuilds).toBe(1); + }); + it('demotes retention when any input in a parallel batch is unsafe', async () => { const { vmBuilds, result } = await drive( 'wrun_retained_mixed_batch', diff --git a/packages/core/src/runtime.ts b/packages/core/src/runtime.ts index ab6948d1b9..846694bbde 100644 --- a/packages/core/src/runtime.ts +++ b/packages/core/src/runtime.ts @@ -340,6 +340,13 @@ function openHookAndWaitState(events: Event[]): { * The open hook/wait scan is O(events), so it is taken through a lazy getter * and consulted last, after every cheap check has passed. * + * INVARIANT this predicate leans on: every suspension signaler that does NOT + * carry the step-consumer generation guard (sleep, hook, attribute — see + * `suspensionGeneration` in private.ts) must be unretainable here, either via + * a non-step queue item or the open hook/wait scan. A new signaler that + * satisfies neither would let a stale signal be accepted as a fresh + * suspension on a resumed session. + * * Quiescence assumes workflow code stays inside the sandbox's determinism * contract. Escaping to the host realm (e.g. recovering the host `Function` * constructor from an exposed host class to schedule real timers) makes a From 5bc0d1120a209576589b0e07e0d9f1dff2d9f94c Mon Sep 17 00:00:00 2001 From: Nathan Colosimo <110621881+NathanColosimo@users.noreply.github.com> Date: Wed, 29 Jul 2026 15:59:11 -0700 Subject: [PATCH 39/43] simplify workflow.ts types: 5 named types (WorkflowResult/WorkflowResumeResult), async resume(), rename runtime local to retainedSession --- packages/core/src/runtime.ts | 37 ++++----- packages/core/src/workflow.ts | 140 ++++++++++++++++------------------ 2 files changed, 78 insertions(+), 99 deletions(-) diff --git a/packages/core/src/runtime.ts b/packages/core/src/runtime.ts index 846694bbde..0ba2923a27 100644 --- a/packages/core/src/runtime.ts +++ b/packages/core/src/runtime.ts @@ -103,7 +103,7 @@ import { buildWorkflowSuspensionMessage } from './util.js'; import { replayWorkflow, resumeWorkflow, - type WorkflowExecutionResult, + type WorkflowResumeResult, type WorkflowSession, } from './workflow.js'; @@ -1235,12 +1235,10 @@ export function workflowEntrypoint( encryptionKey ); - let workflowExecution: - | { readonly type: 'replay' } - | { - readonly type: 'retained'; - readonly session: WorkflowSession; - } = { type: 'replay' }; + // The live VM parked at the previous boundary, when the + // retention decision kept it. null → this iteration cold- + // replays. Invocation-scoped: dies with this delivery. + let retainedSession: WorkflowSession | null = null; // Main replay loop // biome-ignore lint/correctness/noConstantCondition: intentional loop @@ -1574,7 +1572,7 @@ export function workflowEntrypoint( workflowRunId: runId, loopIteration, eventCount: events.length, - executionMode: workflowExecution.type, + executionMode: retainedSession ? 'retained' : 'replay', }); replayStart = Date.now(); // Start every missing decrypt/decompress operation up @@ -1586,16 +1584,12 @@ export function workflowEntrypoint( workflowRun, events ); - let workflowResult: WorkflowExecutionResult = - workflowExecution.type === 'retained' - ? await resumeWorkflow( - workflowExecution.session, - events - ) - : { type: 'replay' }; + let workflowResult: WorkflowResumeResult = retainedSession + ? await resumeWorkflow(retainedSession, events) + : { type: 'replay' }; if (workflowResult.type === 'replay') { - workflowExecution = { type: 'replay' }; + retainedSession = null; workflowResult = await replayWorkflow({ workflowCode, workflowRun, @@ -1615,10 +1609,7 @@ export function workflowEntrypoint( // Park the live session; the suspension catch below // makes the one retention decision — keep it for the // next iteration or discard it for a fresh replay. - workflowExecution = { - type: 'retained', - session: workflowResult.session, - }; + retainedSession = workflowResult.session; throw workflowResult.suspension; } @@ -1627,7 +1618,7 @@ export function workflowEntrypoint( workflowRunId: runId, loopIteration, replayMs: Date.now() - replayStart, - executionMode: workflowExecution.type, + executionMode: retainedSession ? 'retained' : 'replay', }); // Workflow completed. Send the snapshot but do NOT @@ -1870,14 +1861,14 @@ export function workflowEntrypoint( // out-of-band continuation source and provably // passive step inputs. if ( - workflowExecution.type === 'retained' && + retainedSession && !canRetainWorkflowSession( err, suspensionResult.retainedStepInputsSafe, getOpenHookWaitState ) ) { - workflowExecution = { type: 'replay' }; + retainedSession = null; } preStepBlockingMs += suspensionResult.hookCreationMs; diff --git a/packages/core/src/workflow.ts b/packages/core/src/workflow.ts index 4e44628a60..be5284947b 100644 --- a/packages/core/src/workflow.ts +++ b/packages/core/src/workflow.ts @@ -117,33 +117,7 @@ async function drainPendingQueueItems( } } -interface WorkflowCompletion { - readonly output: unknown; - readonly resultType: string; -} - -type WorkflowSessionState = - | { - readonly type: 'running'; - readonly interruption: PromiseWithResolvers; - } - | { readonly type: 'suspended'; readonly suspension: WorkflowSuspension } - | { readonly type: 'replay' } - | { readonly type: 'completed' }; - -export interface WorkflowSession { - readonly workflowRun: WorkflowRun; - readonly argumentCount: number; - resume(events: Event[]): WorkflowSessionResumeResult; -} - -type WorkflowSessionResumeResult = - | { - readonly type: 'resumed'; - readonly execution: Promise; - } - | { readonly type: 'replay' }; - +/** Everything needed to cold-start a workflow VM over an event log. */ interface WorkflowSessionOptions { readonly workflowCode: string; readonly workflowRun: WorkflowRun; @@ -154,55 +128,79 @@ interface WorkflowSessionOptions { readonly worldCapabilities?: WorldCapabilities; } +/** + * A live workflow VM, parked at a suspension boundary. `resume` advances it + * by appending events instead of replaying from scratch. + */ +export interface WorkflowSession { + readonly workflowRun: WorkflowRun; + readonly argumentCount: number; + resume(events: Event[]): Promise; +} + /** A finished execution attempt: the workflow's output or a live boundary. */ -type WorkflowOutcome = - | { readonly type: 'completed'; readonly output: unknown } +export type WorkflowResult = + | { + readonly type: 'completed'; + readonly output: unknown; + /** `typeof` the raw return value, for telemetry. */ + readonly resultType: string; + } | { readonly type: 'suspended'; readonly suspension: WorkflowSuspension; readonly session: WorkflowSession; }; -/** `resumeWorkflow`'s result: `replay` means "cold-replay me instead". */ -export type WorkflowExecutionResult = - | WorkflowOutcome - | { readonly type: 'replay' }; - /** - * Cold-start: build a fresh VM session and replay it over `events`. A fresh - * replay always completes or suspends. + * `resume` can additionally decline — `{ type: 'replay' }` means "this + * session is unusable, cold-replay instead". A fresh replay never declines. */ +export type WorkflowResumeResult = WorkflowResult | { readonly type: 'replay' }; + +/** The session's private state machine. */ +type WorkflowSessionState = + | { + readonly type: 'running'; + readonly interruption: PromiseWithResolvers; + } + | { readonly type: 'suspended'; readonly suspension: WorkflowSuspension } + | { readonly type: 'replay' } + | { readonly type: 'completed' }; + +/** Cold-start: build a fresh VM session and replay it over `events`. */ export function replayWorkflow( options: WorkflowSessionOptions -): Promise { +): Promise { return traceExecution( 'replay', options.workflowRun, options.events, async (span) => { const { session, execution } = await createWorkflowSession(options); - return awaitOutcome(session, execution, span); + span?.setAttributes({ + ...Attribute.WorkflowArgumentsCount(session.argumentCount), + }); + return recordResult(await execution, span); } ); } -/** - * Warm-start: advance a retained session by appending events. Returns - * `{ type: 'replay' }` when the session declines (demoted, or `events` is - * not a strict extension of what it already consumed). - */ +/** Warm-start: advance a retained session by appending events. */ export function resumeWorkflow( session: WorkflowSession, events: Event[] -): Promise { +): Promise { return traceExecution( 'retained', session.workflowRun, events, async (span) => { - const resumed = session.resume(events); - if (resumed.type === 'replay') return resumed; - return awaitOutcome(session, resumed.execution, span); + span?.setAttributes({ + ...Attribute.WorkflowArgumentsCount(session.argumentCount), + }); + const result = await session.resume(events); + return result.type === 'replay' ? result : recordResult(result, span); } ); } @@ -225,28 +223,18 @@ function traceExecution( }); } -/** Await the execution attempt, converting a suspension into an outcome. */ -async function awaitOutcome( - session: WorkflowSession, - execution: Promise, +function recordResult( + result: WorkflowResult, span: Span | undefined -): Promise { - span?.setAttributes({ - ...Attribute.WorkflowArgumentsCount(session.argumentCount), - }); - try { - const completed = await execution; +): WorkflowResult { + if (result.type === 'completed') { span?.setAttributes({ - ...Attribute.WorkflowResultType(completed.resultType), + ...Attribute.WorkflowResultType(result.resultType), }); - return { type: 'completed', output: completed.output }; - } catch (error) { - if (WorkflowSuspension.is(error)) { - if (span) applyWorkflowSuspensionToSpan(error, span); - return { type: 'suspended', suspension: error, session }; - } - throw error; + } else if (span) { + applyWorkflowSuspensionToSpan(result.suspension, span); } + return result; } /** @@ -302,7 +290,7 @@ async function createWorkflowSession({ worldCapabilities, }: WorkflowSessionOptions): Promise<{ session: WorkflowSession; - execution: Promise; + execution: Promise; }> { const startedAt = workflowRun.startedAt; if (!startedAt) { @@ -1007,7 +995,10 @@ async function createWorkflowSession({ }); await workflowContext.promiseQueue; - const workflowExecution = (async (): Promise => { + // The user function's promise. It may stay pending across many resumes + // (each parked step promise holds it up) and is raced against the current + // attempt's interruption in waitForExecution. + const workflowBody = (async (): Promise => { return await workflowFn(...args); })(); @@ -1038,13 +1029,13 @@ async function createWorkflowSession({ const waitForExecution = async ( interruption: PromiseWithResolvers - ): Promise => { + ): Promise => { let result: unknown; try { - result = await Promise.race([workflowExecution, interruption.promise]); + result = await Promise.race([workflowBody, interruption.promise]); } catch (error) { if (state.type === 'suspended' && error === state.suspension) { - throw error; + return { type: 'suspended', suspension: state.suspension, session }; } return failWorkflow(error); } @@ -1071,7 +1062,7 @@ async function createWorkflowSession({ runReadyBarrier ); - return { output, resultType: typeof result }; + return { type: 'completed', output, resultType: typeof result }; } catch (error) { return failWorkflow(error); } @@ -1080,7 +1071,7 @@ async function createWorkflowSession({ const session: WorkflowSession = { workflowRun, argumentCount: args.length, - resume(nextEvents) { + async resume(nextEvents) { switch (state.type) { case 'suspended': { // The full O(known events) prefix compare is required: the runtime @@ -1102,10 +1093,7 @@ async function createWorkflowSession({ state = { type: 'running', interruption }; workflowContext.suspensionGeneration++; eventsConsumer.append(nextEvents.slice(knownEvents.length)); - return { - type: 'resumed', - execution: waitForExecution(interruption), - }; + return waitForExecution(interruption); } case 'replay': return { type: 'replay' }; From e1e539919641fb61963c0bf2836f9ee1932b8076 Mon Sep 17 00:00:00 2001 From: Nathan Colosimo <110621881+NathanColosimo@users.noreply.github.com> Date: Thu, 30 Jul 2026 09:43:18 -0700 Subject: [PATCH 40/43] add retention-interleaving e2e (retained/demoted/wait/hook boundaries), drop session telemetry test --- packages/core/e2e/e2e.test.ts | 27 ++++++++ .../src/workflow-session-telemetry.test.ts | 68 ------------------- workbench/example/workflows/99_e2e.ts | 42 ++++++++++++ 3 files changed, 69 insertions(+), 68 deletions(-) delete mode 100644 packages/core/src/workflow-session-telemetry.test.ts diff --git a/packages/core/e2e/e2e.test.ts b/packages/core/e2e/e2e.test.ts index 9adc100b3e..a2d7e0cc04 100644 --- a/packages/core/e2e/e2e.test.ts +++ b/packages/core/e2e/e2e.test.ts @@ -443,6 +443,33 @@ describe('e2e', () => { expect(returnValue).toBe('B'); }); + // Interleaves VM-retention modes: retained boundaries (primitive step + // args), demoted boundaries (object args), sleeps, a step-vs-sleep race, + // and a hook resolved in parallel with a step. Asserts the exact composite + // result so a dropped/duplicated/misordered boundary fails loudly. + test('retainedInterleavingWorkflow', { timeout: 90_000 }, async () => { + const token = Math.random().toString(36).slice(2); + const run = await start(await e2e('retainedInterleavingWorkflow'), [ + token, + ]); + + const hook = await waitForHook(token, { runId: run.runId }); + await resumeHook(hook, { delta: 5 }); + + const returnValue = await run.returnValue; + expect(returnValue).toEqual({ + a: 3, + b: 3, + c: 13, + d: 23, + e: 13, + f: 24, + winner: 'step', + g: 137, + h: 142, + }); + }); + test.skipIf(!isNext)( 'importedStepOnlyWorkflow', { timeout: 60_000 }, diff --git a/packages/core/src/workflow-session-telemetry.test.ts b/packages/core/src/workflow-session-telemetry.test.ts deleted file mode 100644 index f2305c794c..0000000000 --- a/packages/core/src/workflow-session-telemetry.test.ts +++ /dev/null @@ -1,68 +0,0 @@ -import { trace as otelTrace } from '@opentelemetry/api'; -import { - BasicTracerProvider, - InMemorySpanExporter, - SimpleSpanProcessor, -} from '@opentelemetry/sdk-trace-base'; -import type { WorkflowRun } from '@workflow/world'; -import { afterAll, afterEach, beforeAll, describe, expect, it } from 'vitest'; -import { ReplayPayloadCache } from './replay-payload-cache.js'; -import { dehydrateWorkflowArguments } from './serialization.js'; -import { replayWorkflow } from './workflow.js'; - -const exporter = new InMemorySpanExporter(); -const provider = new BasicTracerProvider(); - -beforeAll(() => { - provider.addSpanProcessor(new SimpleSpanProcessor(exporter)); - otelTrace.setGlobalTracerProvider(provider); -}); - -afterAll(async () => { - await provider.shutdown(); - otelTrace.disable(); -}); - -afterEach(() => { - exporter.reset(); -}); - -describe('retained workflow telemetry', () => { - it('records suspension attributes on workflow.run spans', async () => { - const runId = 'wrun_retained_telemetry'; - const workflowRun: WorkflowRun = { - runId, - workflowName: 'workflow', - status: 'running', - input: await dehydrateWorkflowArguments([], runId, undefined, []), - createdAt: new Date('2024-01-01T00:00:00.000Z'), - updatedAt: new Date('2024-01-01T00:00:00.000Z'), - startedAt: new Date('2024-01-01T00:00:00.000Z'), - deploymentId: 'test-deployment', - }; - const workflowCode = ` - const step = globalThis[Symbol.for("WORKFLOW_USE_STEP")]("step"); - async function workflow() { await step(); } - globalThis.__private_workflows = new Map([["workflow", workflow]]);`; - - const result = await replayWorkflow({ - workflowCode, - workflowRun, - events: [], - encryptionKey: undefined, - replayPayloadCache: new ReplayPayloadCache(undefined), - }); - expect(result.type).toBe('suspended'); - - const span = exporter - .getFinishedSpans() - .find((candidate) => candidate.name === 'workflow.run workflow'); - expect(span?.attributes).toMatchObject({ - 'workflow.execution.mode': 'replay', - 'workflow.suspension.state': 'suspended', - 'workflow.suspension.step_count': 1, - 'workflow.suspension.hook_count': 0, - 'workflow.suspension.wait_count': 0, - }); - }); -}); diff --git a/workbench/example/workflows/99_e2e.ts b/workbench/example/workflows/99_e2e.ts index 2fb89e129d..0ef0bcc661 100644 --- a/workbench/example/workflows/99_e2e.ts +++ b/workbench/example/workflows/99_e2e.ts @@ -246,6 +246,48 @@ export async function stepWinsRaceWorkflow() { ////////////////////////////////////////////////////////// +// Takes an OBJECT argument: under VM retention (WORKFLOW_RETAINED_VM), a +// boundary whose new step has a non-primitive input falls back to cold +// replay instead of resuming the retained VM. +async function unwrapValue(box: { value: number }) { + 'use step'; + return box.value; +} + +/** + * Interleaves every retention mode the runtime can hit: retained boundaries + * (primitive step args), demoted boundaries (object args), wait boundaries + * (sleep, step-vs-sleep race), and a hook awaited in parallel with a step. + * The chained arithmetic makes any dropped, duplicated, or misordered + * boundary visible in the final output. + */ +export async function retainedInterleavingWorkflow(token: string) { + 'use workflow'; + // Retained: sequential primitive-arg step. + const a = await add(1, 2); // 3 + // Demoted: object argument. + const b = await unwrapValue({ value: a }); // 3 + // Retained: parallel all-primitive batch. + const [c, d] = await Promise.all([add(b, 10), add(b, 20)]); // 13, 23 + // Demoted: mixed parallel batch (one object arg, one primitive). + const [e, f] = await Promise.all([unwrapValue({ value: c }), add(d, 1)]); // 13, 24 + // Wait boundary: step races (and beats) a sleep. + const winner = await Promise.race([ + delayMsStep(100, 'step'), + sleep('30s').then(() => 'sleep'), + ]); // 'step' + // Wait boundary: plain sleep. + await sleep('1s'); + // Hook boundary: hook payload awaited in parallel with a primitive step. + using hook = createHook<{ delta: number }>({ token }); + const [payload, g] = await Promise.all([hook, add(e + f, 100)]); // _, 137 + // Retained again after all the demotions. + const h = await add(g, payload.delta); // 137 + delta + return { a, b, c, d, e, f, winner, g, h }; +} + +////////////////////////////////////////////////////////// + async function nullByteStep() { 'use step'; return 'null byte \0'; From e0dea1422d62a11651b80a86c3e710d3bf155b22 Mon Sep 17 00:00:00 2001 From: Nathan Colosimo <110621881+NathanColosimo@users.noreply.github.com> Date: Mon, 3 Aug 2026 10:20:09 -0700 Subject: [PATCH 41/43] discard the retained session on every in-process 412 restart MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Review finding (both panel reviewers): restartReplayInProcess — added on main by #3145 while this branch was in flight — reset the cached log but not the parked VM session. Any stale-snapshot continue then resumed a session belonging to the discarded log: after a run_completed 412 the completed session's resume() throws and the run is durably failed despite having completed; after a suspension-create 412 the session is resumed without ever passing the retention decision, bypassing both the WORKFLOW_RETAINED_VM kill switch and the step-input gate. A restart now always falls back to a fresh replay. Regression test injects a 412 on run_completed and proves fresh-replay completion (red without the fix). --- packages/core/src/retained-vm-loop.test.ts | 27 +++++++++++++++++++++- packages/core/src/runtime.ts | 8 +++++++ 2 files changed, 34 insertions(+), 1 deletion(-) diff --git a/packages/core/src/retained-vm-loop.test.ts b/packages/core/src/retained-vm-loop.test.ts index 94a9cd4e8d..011e8fc6a8 100644 --- a/packages/core/src/retained-vm-loop.test.ts +++ b/packages/core/src/retained-vm-loop.test.ts @@ -1,3 +1,4 @@ +import { PreconditionFailedError } from '@workflow/errors'; import { type Event, SPEC_VERSION_CURRENT, @@ -131,7 +132,12 @@ registerStepFunction('r_echo', async (value) => value); // Drive the full workflow handler over a stateful (dynamic) event log so the // inline loop makes real progress across its own writes, exactly like a World. // Non-turbo (no runInput, attempt 2) to keep the path simple and deterministic. -async function drive(runId: string, workflowCode = twoStepWorkflow) { +async function drive( + runId: string, + workflowCode = twoStepWorkflow, + options: { failEventTypeOnce?: string } = {} +) { + let { failEventTypeOnce } = options; const run: WorkflowRun = { runId, workflowName: 'workflow', @@ -147,6 +153,10 @@ async function drive(runId: string, workflowCode = twoStepWorkflow) { let seq = 0; const eventsCreate = vi.fn(async (_runId: string, data: any) => { + if (data.eventType === failEventTypeOnce) { + failEventTypeOnce = undefined; + throw new PreconditionFailedError('stale snapshot (test-injected)'); + } createdEvents.push(data); if (data.eventType === 'run_started') { return { run, events }; @@ -293,6 +303,21 @@ describe('retained VM through the inline replay loop', () => { expect(vmBuilds).toBeGreaterThan(1); }); + it('discards the retained session when a 412 forces an in-process restart', async () => { + // A stale-snapshot rejection of run_completed restarts the replay in + // process (see restartReplayInProcess). The parked session belongs to the + // discarded log — resuming it would replay a completed session (throw → + // run_failed) or bypass the retention decision entirely. The restart must + // fall back to a fresh replay and still complete the run. + const { vmBuilds, result } = await drive( + 'wrun_retained_412_restart', + twoStepWorkflow, + { failEventTypeOnce: 'run_completed' } + ); + expect(result).toBe(30); + expect(vmBuilds).toBeGreaterThan(1); + }); + it('retains a VM that used the synchronous crypto.subtle.digest', async () => { const { vmBuilds, result } = await drive( 'wrun_retained_digest', diff --git a/packages/core/src/runtime.ts b/packages/core/src/runtime.ts index e40dc6c517..27a322ef71 100644 --- a/packages/core/src/runtime.ts +++ b/packages/core/src/runtime.ts @@ -1045,6 +1045,14 @@ export function workflowEntrypoint( return false; } preconditionRestarts++; + // Every stale-snapshot restart invalidates the parked VM: + // the log it consumed was missing events, so only a fresh + // replay over the corrected log is authoritative. This + // also covers the suspension-create 412, which fires + // before the retention decision (kill switch + step-input + // gate) ever ran, and the run_completed 412, where a + // completed session must not be resumed again. + retainedSession = null; // A World MAY return the events we were missing on the 412. // Trust it only on the FIRST restart: its completeness proof // leans on the backend's own bookkeeping, so if that From e7cfaa7364931422ce595f2e15044f56a0e9a912 Mon Sep 17 00:00:00 2001 From: Nathan Colosimo <110621881+NathanColosimo@users.noreply.github.com> Date: Mon, 3 Aug 2026 11:32:43 -0700 Subject: [PATCH 42/43] =?UTF-8?q?review=20round=202:=20set=20suspensionGen?= =?UTF-8?q?eration=20in=20typed=20test=20harness=20contexts;=20correct=20t?= =?UTF-8?q?he=20open-hook/wait=20scan=20comment=20(this=20suspension's=20w?= =?UTF-8?q?rites=20are=20not=20merged=20into=20the=20cached=20log=20?= =?UTF-8?q?=E2=80=94=20non-step=20suspensions=20never=20reach=20the=20scan?= =?UTF-8?q?)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- packages/core/src/delivery-barrier-coverage.test.ts | 1 + packages/core/src/hook-sleep-interaction.test.ts | 1 + packages/core/src/runtime.ts | 8 ++++++-- packages/core/src/step-delivery-hop-count.test.ts | 1 + packages/core/src/step-delivery-ordering.test.ts | 1 + packages/core/src/workflow/sleep.test.ts | 1 + 6 files changed, 11 insertions(+), 2 deletions(-) diff --git a/packages/core/src/delivery-barrier-coverage.test.ts b/packages/core/src/delivery-barrier-coverage.test.ts index 836d7d3948..44248742e6 100644 --- a/packages/core/src/delivery-barrier-coverage.test.ts +++ b/packages/core/src/delivery-barrier-coverage.test.ts @@ -72,6 +72,7 @@ function setupWorkflowContext(events: Event[]): WorkflowOrchestratorContext { const promiseQueueHolder = { current: Promise.resolve() }; const ctxRef: { current?: WorkflowOrchestratorContext } = {}; const ctx: WorkflowOrchestratorContext = { + suspensionGeneration: 0, runId: 'wrun_test', encryptionKey: undefined, replayPayloadCache: new ReplayPayloadCache(undefined), diff --git a/packages/core/src/hook-sleep-interaction.test.ts b/packages/core/src/hook-sleep-interaction.test.ts index 29d64fcc58..135cea5056 100644 --- a/packages/core/src/hook-sleep-interaction.test.ts +++ b/packages/core/src/hook-sleep-interaction.test.ts @@ -42,6 +42,7 @@ function setupWorkflowContext(events: Event[]): WorkflowOrchestratorContext { // silently dropping them. const ctxRef: { current?: WorkflowOrchestratorContext } = {}; const ctx: WorkflowOrchestratorContext = { + suspensionGeneration: 0, runId: 'wrun_test', encryptionKey: undefined, replayPayloadCache: new ReplayPayloadCache(undefined), diff --git a/packages/core/src/runtime.ts b/packages/core/src/runtime.ts index 27a322ef71..33c2854010 100644 --- a/packages/core/src/runtime.ts +++ b/packages/core/src/runtime.ts @@ -2431,8 +2431,12 @@ export function workflowEntrypoint( } eventsCursor = suspensionLog.cursor; - // Open hooks/waits in the cumulative log (this - // suspension's writes are already merged in). Computed + // Open hooks/waits in the log as loaded for this + // replay. This suspension's own hook/wait writes are + // NOT in it — they never reach retention anyway, + // because a suspension containing a non-step item + // fails canRetainWorkflowSession's type check before + // the scan is consulted. Computed // lazily, at most once, and shared between the // retention decision here and the delta/turbo gates // below — the attr-detour and hook-conflict paths diff --git a/packages/core/src/step-delivery-hop-count.test.ts b/packages/core/src/step-delivery-hop-count.test.ts index f7ea3e09b1..7f9ce88134 100644 --- a/packages/core/src/step-delivery-hop-count.test.ts +++ b/packages/core/src/step-delivery-hop-count.test.ts @@ -55,6 +55,7 @@ function setupWorkflowContext( const promiseQueueHolder = { current: Promise.resolve() }; const ctxRef: { current?: WorkflowOrchestratorContext } = {}; const ctx: WorkflowOrchestratorContext = { + suspensionGeneration: 0, runId: 'wrun_test', encryptionKey: undefined, replayPayloadCache, diff --git a/packages/core/src/step-delivery-ordering.test.ts b/packages/core/src/step-delivery-ordering.test.ts index 85933b7e7c..5bf7257047 100644 --- a/packages/core/src/step-delivery-ordering.test.ts +++ b/packages/core/src/step-delivery-ordering.test.ts @@ -102,6 +102,7 @@ function setupWorkflowContext( const promiseQueueHolder = { current: Promise.resolve() }; const ctxRef: { current?: WorkflowOrchestratorContext } = {}; const ctx: WorkflowOrchestratorContext = { + suspensionGeneration: 0, runId: 'wrun_test', encryptionKey: undefined, replayPayloadCache, diff --git a/packages/core/src/workflow/sleep.test.ts b/packages/core/src/workflow/sleep.test.ts index a7c4723dd7..79c2202264 100644 --- a/packages/core/src/workflow/sleep.test.ts +++ b/packages/core/src/workflow/sleep.test.ts @@ -20,6 +20,7 @@ function setupWorkflowContext(events: Event[]): WorkflowOrchestratorContext { const ulid = monotonicFactory(() => context.globalThis.Math.random()); const workflowStartedAt = context.globalThis.Date.now(); const ctx: WorkflowOrchestratorContext = { + suspensionGeneration: 0, runId: 'wrun_test', encryptionKey: undefined, replayPayloadCache: new ReplayPayloadCache(undefined), From eec9a0928cf6226d0335a12e44d6166409f39608 Mon Sep 17 00:00:00 2001 From: Nathan Colosimo <110621881+NathanColosimo@users.noreply.github.com> Date: Mon, 3 Aug 2026 12:00:43 -0700 Subject: [PATCH 43/43] simplify pass: reuse once() from @workflow/utils for the open-hook/wait memo; drop optional-chaining that contradicted the surrounding guards --- packages/core/src/retained-vm-loop.test.ts | 6 +++--- packages/core/src/runtime.ts | 19 ++++++++----------- packages/core/src/workflow.ts | 2 +- 3 files changed, 12 insertions(+), 15 deletions(-) diff --git a/packages/core/src/retained-vm-loop.test.ts b/packages/core/src/retained-vm-loop.test.ts index 011e8fc6a8..9e8beeef6f 100644 --- a/packages/core/src/retained-vm-loop.test.ts +++ b/packages/core/src/retained-vm-loop.test.ts @@ -177,15 +177,15 @@ async function drive( step: { runId, stepId: data.correlationId, - stepName: d?.stepName, + stepName: d.stepName, status: 'running' as const, attempt: 1, - input: d?.input, + input: d.input, startedAt: new Date(), createdAt: new Date(), updatedAt: new Date(), }, - ...(d?.input !== undefined ? { stepCreated: true } : {}), + ...(d.input !== undefined ? { stepCreated: true } : {}), }; } return { event }; diff --git a/packages/core/src/runtime.ts b/packages/core/src/runtime.ts index 33c2854010..b34df4e209 100644 --- a/packages/core/src/runtime.ts +++ b/packages/core/src/runtime.ts @@ -12,7 +12,7 @@ import { RunExpiredError, WorkflowRuntimeError, } from '@workflow/errors'; -import { setWorkflowBasePath } from '@workflow/utils'; +import { once, setWorkflowBasePath } from '@workflow/utils'; import { parseWorkflowName, workflowDisplayName, @@ -509,7 +509,7 @@ function openHookAndWaitState(events: Event[]): { function canRetainWorkflowSession( suspension: WorkflowSuspension, stepInputsSafe: boolean, - getOpenHookWaitState: () => ReturnType + openHookWait: { value: ReturnType } ): boolean { if ( !isVmRetentionEnabled() || @@ -519,7 +519,7 @@ function canRetainWorkflowSession( ) { return false; } - const { openHook, openWait } = getOpenHookWaitState(); + const { openHook, openWait } = openHookWait.value; return !openHook && !openWait; } @@ -2446,12 +2446,9 @@ export function workflowEntrypoint( // lose `cachedEvents`'s non-null narrowing. Nothing in // this catch scope reassigns the array. const suspensionEvents = cachedEvents; - let openHookWaitMemo: - | ReturnType - | undefined; - const getOpenHookWaitState = () => - (openHookWaitMemo ??= - openHookAndWaitState(suspensionEvents)); + const openHookWait = once(() => + openHookAndWaitState(suspensionEvents) + ); // The single retention decision: keep the parked // session only across a pure step boundary with no @@ -2462,7 +2459,7 @@ export function workflowEntrypoint( !canRetainWorkflowSession( err, suspensionResult.retainedStepInputsSafe, - getOpenHookWaitState + openHookWait ) ) { retainedSession = null; @@ -2787,7 +2784,7 @@ export function workflowEntrypoint( // Open hooks/waits are consulted by all three gates // below; resolve the memoized scan once here. - const openHookWaitState = getOpenHookWaitState(); + const openHookWaitState = openHookWait.value; // Inline-delta fast path gate. We request the delta — // and on the next iteration consume it in place of the diff --git a/packages/core/src/workflow.ts b/packages/core/src/workflow.ts index d50b439d07..113836e91d 100644 --- a/packages/core/src/workflow.ts +++ b/packages/core/src/workflow.ts @@ -1087,7 +1087,7 @@ async function createWorkflowSession({ const isStrictExtension = nextEvents.length > knownEvents.length && knownEvents.every( - (event, index) => event.eventId === nextEvents[index]?.eventId + (event, index) => event.eventId === nextEvents[index].eventId ); if (!isStrictExtension) { state = { type: 'replay' };