diff --git a/.changeset/resume-partial-replay-streams.md b/.changeset/resume-partial-replay-streams.md new file mode 100644 index 0000000000..1aba3274fe --- /dev/null +++ b/.changeset/resume-partial-replay-streams.md @@ -0,0 +1,5 @@ +--- +"@workflow/world-vercel": patch +--- + +Resume interrupted replay event streams after their last validated event without repeating accepted event writes. diff --git a/packages/world-vercel/src/event-retry.test.ts b/packages/world-vercel/src/event-retry.test.ts index 226888dbc0..ae91530f15 100644 --- a/packages/world-vercel/src/event-retry.test.ts +++ b/packages/world-vercel/src/event-retry.test.ts @@ -9,6 +9,7 @@ import { EventTypeSchema } from '@workflow/world'; import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest'; import { EVENT_RETRY_ELIGIBILITY, + EventPostResponseError, isRetryableEventPostError, MAX_EVENT_POST_RETRIES, THROTTLE_RETRY_BUDGET_MS, @@ -103,6 +104,19 @@ describe('isRetryableEventPostError', () => { ).toBe(true); }); + it('does not repeat a POST to repair a failed response continuation', () => { + const transport = new WorkflowWorldError('continuation failed', { + code: 'TRANSPORT', + }); + expect( + isRetryableEventPostError( + new EventPostResponseError('POST response already accepted', { + cause: transport, + }) + ) + ).toBe(false); + }); + it('retries a TRANSPORT failure from either transport', () => { // The one code that lets this policy serve HTTP and WS alike: `utils.ts` // sets it for a `fetch` that failed transiently, `events-v4.ts` for a WS diff --git a/packages/world-vercel/src/event-retry.ts b/packages/world-vercel/src/event-retry.ts index 3228bd45b8..30dbf6a97e 100644 --- a/packages/world-vercel/src/event-retry.ts +++ b/packages/world-vercel/src/event-retry.ts @@ -76,6 +76,18 @@ import type { z } from 'zod'; * `hook_conflict`, which the SDK never POSTs). */ type WorkflowEventType = z.infer; +/** + * A follow-up read failed after the event POST had already succeeded and + * exposed validated response data. Repeating the mutation cannot repair that + * read and would redownload/re-observe the accepted prefix. + */ +export class EventPostResponseError extends Error { + constructor(message: string, options?: ErrorOptions) { + super(message, options); + this.name = 'EventPostResponseError'; + } +} + export interface EventRetryPolicy { /** Whether a failed POST of this event type may be retried in-process. */ retryable: boolean; @@ -237,14 +249,8 @@ function collectErrorMarkers(err: unknown, depth = 0): string[] { return markers; } -/** - * Whether a failed event POST should be retried as a transient failure. - * Retries transient/ambiguous transport failures and transient 5xx; never - * retries a definitive response (409/410/425/429 and other 4xx). 429 is not - * "transient" in this classification — `withEventPostRetry` gives it its own - * budgeted, Retry-After-honoring policy. - */ -export function isRetryableEventPostError(err: unknown): boolean { +/** Transient request failures shared by idempotent event reads and writes. */ +export function isRetryableEventRequestError(err: unknown): boolean { // Definitive, server-considered outcomes — never retried as *transient*. // (425 is left to the runtime's retry-after handling; 429 has its own // in-process policy in withEventPostRetry, gated by THROTTLE_RETRY_BUDGET_MS @@ -259,9 +265,6 @@ export function isRetryableEventPostError(err: unknown): boolean { } if (WorkflowWorldError.is(err)) { - // Body parsed past the response but the write may have landed — safe to - // retry for eligible events (a landed original re-surfaces as 409). - if (err.code === 'PARSE_ERROR') return true; // A transport failure the world layer already classified as transient: // `utils.ts` sets this for a `fetch` failing with a // `TRANSIENT_TRANSPORT_ERROR_CODES` code, `events-v4.ts` for a WS socket @@ -289,6 +292,18 @@ export function isRetryableEventPostError(err: unknown): boolean { return collectErrorMarkers(err).some((m) => TRANSIENT_CODES.has(m)); } +/** + * Whether a failed event POST should be retried as a transient failure. + * Response parse errors are write-specific: the mutation may have landed, and + * eligible event types converge when it is repeated. A follow-up read failure + * after an accepted POST is explicitly excluded. + */ +export function isRetryableEventPostError(err: unknown): boolean { + if (err instanceof EventPostResponseError) return false; + if (WorkflowWorldError.is(err) && err.code === 'PARSE_ERROR') return true; + return isRetryableEventRequestError(err); +} + const sleep = (ms: number): Promise => new Promise((resolve) => setTimeout(resolve, ms)); diff --git a/packages/world-vercel/src/events-v4.test.ts b/packages/world-vercel/src/events-v4.test.ts index 911e1a57e3..6a2e333df0 100644 --- a/packages/world-vercel/src/events-v4.test.ts +++ b/packages/world-vercel/src/events-v4.test.ts @@ -21,14 +21,22 @@ import { getWorkflowRunEventsV4, throwForErrorResponse, } from './events-v4.js'; -import { encodeFrame, V4_FRAME_CONTENT_TYPE } from './frames.js'; import { - EVENTS_RECYCLE_AFTER_CONSECUTIVE_FAILURES, - getEventsDispatcher, -} from './http-client.js'; + encodeFrame, + InvalidFrameError, + V4_FRAME_CONTENT_TYPE, +} from './frames.js'; +import { getEventsDispatcher } from './http-client.js'; import { WORKFLOW_SERVER_URL_OVERRIDE } from './utils.js'; const CREATED_AT = '2026-06-10T00:00:00.000Z'; +const ORIGIN = WORKFLOW_SERVER_URL_OVERRIDE || 'https://vercel-workflow.com'; + +function mockAgent() { + const agent = new MockAgent(); + agent.disableNetConnect(); + return agent; +} function createEventBody( event: AnyEventRequest, @@ -264,10 +272,7 @@ describe('throwForErrorResponse', () => { */ describe('getWorkflowRunEventsV4 over HTTP', () => { it('parses a frame stream fetched via a custom dispatcher', async () => { - const origin = - WORKFLOW_SERVER_URL_OVERRIDE || 'https://vercel-workflow.com'; - const agent = new MockAgent(); - agent.disableNetConnect(); + const agent = mockAgent(); const body = new TextEncoder().encode('payload-bytes'); const frames = Buffer.concat([ @@ -292,9 +297,9 @@ describe('getWorkflowRunEventsV4 over HTTP', () => { ]); agent - .get(origin) + .get(ORIGIN) .intercept({ - path: '/api/v4/runs/wrun_1/events?returnAll=true', + path: '/api/v4/runs/wrun_1/events?remoteRefBehavior=resolve&returnAll=true', method: 'GET', }) .reply(200, frames, { @@ -302,13 +307,12 @@ describe('getWorkflowRunEventsV4 over HTTP', () => { }); const result = await getWorkflowRunEventsV4( - 'wrun_1', - {}, + { runId: 'wrun_1' }, { token: 'test-token', dispatcher: agent } ); - expect(result.events).toHaveLength(1); - expect(result.events[0]).toMatchObject({ + expect(result.data).toHaveLength(1); + expect(result.data[0]).toMatchObject({ eventId: 'evnt_1', eventData: { input: body }, }); @@ -320,15 +324,12 @@ describe('getWorkflowRunEventsV4 over HTTP', () => { ['an unknown event type', { eventType: 'future_event', eventData: {} }], ['invalid event metadata', { eventType: 'run_created', eventData: {} }], ])('rejects %s', async (_description, meta) => { - const origin = - WORKFLOW_SERVER_URL_OVERRIDE || 'https://vercel-workflow.com'; - const agent = new MockAgent(); - agent.disableNetConnect(); + const agent = mockAgent(); agent - .get(origin) + .get(ORIGIN) .intercept({ - path: '/api/v4/runs/wrun_1/events?returnAll=true', + path: '/api/v4/runs/wrun_1/events?remoteRefBehavior=resolve&returnAll=true', method: 'GET', }) .reply( @@ -342,19 +343,38 @@ describe('getWorkflowRunEventsV4 over HTTP', () => { await expect( getWorkflowRunEventsV4( - 'wrun_1', - {}, + { runId: 'wrun_1' }, { token: 'test-token', dispatcher: agent } ) ).rejects.toThrow(); agent.assertNoPendingInterceptors(); }); + it('does not retry invalid CBOR frame metadata as a transport failure', async () => { + const agent = mockAgent(); + const nonObjectCborFrame = new Uint8Array([0, 0, 0, 1, 0x01, 0, 0, 0, 0]); + + agent + .get(ORIGIN) + .intercept({ + path: '/api/v4/runs/wrun_1/events?remoteRefBehavior=resolve&returnAll=true', + method: 'GET', + }) + .reply(200, nonObjectCborFrame, { + headers: { 'content-type': V4_FRAME_CONTENT_TYPE }, + }); + + await expect( + getWorkflowRunEventsV4( + { runId: 'wrun_1' }, + { token: 'test-token', dispatcher: agent } + ) + ).rejects.toBeInstanceOf(InvalidFrameError); + agent.assertNoPendingInterceptors(); + }); + it('captures an explicit hasMore from the sentinel, independent of next', async () => { - const origin = - WORKFLOW_SERVER_URL_OVERRIDE || 'https://vercel-workflow.com'; - const agent = new MockAgent(); - agent.disableNetConnect(); + const agent = mockAgent(); // The regression shape: a final page still carries a trailing `next` // cursor (incremental-load resume point) but hasMore is false. @@ -380,9 +400,9 @@ describe('getWorkflowRunEventsV4 over HTTP', () => { ]); agent - .get(origin) + .get(ORIGIN) .intercept({ - path: '/api/v4/runs/wrun_1/events?returnAll=true', + path: '/api/v4/runs/wrun_1/events?remoteRefBehavior=resolve&returnAll=true', method: 'GET', }) .reply(200, frames, { @@ -390,8 +410,7 @@ describe('getWorkflowRunEventsV4 over HTTP', () => { }); const result = await getWorkflowRunEventsV4( - 'wrun_1', - {}, + { runId: 'wrun_1' }, { token: 'test-token', dispatcher: agent } ); @@ -400,10 +419,7 @@ describe('getWorkflowRunEventsV4 over HTTP', () => { }); it('rejects an end frame without hasMore', async () => { - const origin = - WORKFLOW_SERVER_URL_OVERRIDE || 'https://vercel-workflow.com'; - const agent = new MockAgent(); - agent.disableNetConnect(); + const agent = mockAgent(); const frames = encodeFrame( { _end: 1, next: 'cursor-2' }, @@ -411,9 +427,9 @@ describe('getWorkflowRunEventsV4 over HTTP', () => { ); agent - .get(origin) + .get(ORIGIN) .intercept({ - path: '/api/v4/runs/wrun_1/events?returnAll=true', + path: '/api/v4/runs/wrun_1/events?remoteRefBehavior=resolve&returnAll=true', method: 'GET', }) .reply(200, frames, { @@ -422,18 +438,14 @@ describe('getWorkflowRunEventsV4 over HTTP', () => { await expect( getWorkflowRunEventsV4( - 'wrun_1', - {}, + { runId: 'wrun_1' }, { token: 'test-token', dispatcher: agent } ) ).rejects.toThrow(); }); it('throws when the stream ends without the end sentinel (truncated response)', async () => { - const origin = - WORKFLOW_SERVER_URL_OVERRIDE || 'https://vercel-workflow.com'; - const agent = new MockAgent(); - agent.disableNetConnect(); + const agent = mockAgent(); // A complete event frame but NO `{_end: 1}` sentinel — what a response // truncated on a frame boundary looks like. Returning this as a @@ -454,9 +466,9 @@ describe('getWorkflowRunEventsV4 over HTTP', () => { ); agent - .get(origin) + .get(ORIGIN) .intercept({ - path: '/api/v4/runs/wrun_1/events?limit=500', + path: '/api/v4/runs/wrun_1/events?limit=500&remoteRefBehavior=resolve', method: 'GET', }) .reply(200, frames, { @@ -465,23 +477,19 @@ describe('getWorkflowRunEventsV4 over HTTP', () => { await expect( getWorkflowRunEventsV4( - 'wrun_1', - { limit: 500 }, + { runId: 'wrun_1', pagination: { limit: 500 } }, { token: 'test-token', dispatcher: agent } ) ).rejects.toThrow(/end-of-stream sentinel/); }); it('resumes a truncated full stream after its last accepted event', async () => { - const origin = - WORKFLOW_SERVER_URL_OVERRIDE || 'https://vercel-workflow.com'; - const agent = new MockAgent(); - agent.disableNetConnect(); + const agent = mockAgent(); agent - .get(origin) + .get(ORIGIN) .intercept({ - path: '/api/v4/runs/wrun_1/events?returnAll=true', + path: '/api/v4/runs/wrun_1/events?remoteRefBehavior=resolve&returnAll=true', method: 'GET', }) .reply( @@ -503,9 +511,9 @@ describe('getWorkflowRunEventsV4 over HTTP', () => { { headers: { 'content-type': V4_FRAME_CONTENT_TYPE } } ); agent - .get(origin) + .get(ORIGIN) .intercept({ - path: '/api/v4/runs/wrun_1/events?returnAll=true&cursor=eid%3Aevnt_1', + path: '/api/v4/runs/wrun_1/events?cursor=eid%3Aevnt_1&remoteRefBehavior=resolve&returnAll=true', method: 'GET', }) .reply( @@ -529,12 +537,11 @@ describe('getWorkflowRunEventsV4 over HTTP', () => { ); const result = await getWorkflowRunEventsV4( - 'wrun_1', - {}, + { runId: 'wrun_1' }, { token: 'test-token', dispatcher: agent } ); - expect(result.events.map((event) => event.eventId)).toEqual([ + expect(result.data.map((event) => event.eventId)).toEqual([ 'evnt_1', 'evnt_2', ]); @@ -542,6 +549,78 @@ describe('getWorkflowRunEventsV4 over HTTP', () => { expect(result.hasMore).toBe(false); agent.assertNoPendingInterceptors(); }); + + it('stops after three partial-stream recovery retries', async () => { + const agent = mockAgent(); + const paths = [ + '/api/v4/runs/wrun_1/events?remoteRefBehavior=resolve&returnAll=true', + '/api/v4/runs/wrun_1/events?cursor=eid%3Aevnt_1&remoteRefBehavior=resolve&returnAll=true', + '/api/v4/runs/wrun_1/events?cursor=eid%3Aevnt_2&remoteRefBehavior=resolve&returnAll=true', + '/api/v4/runs/wrun_1/events?cursor=eid%3Aevnt_3&remoteRefBehavior=resolve&returnAll=true', + ]; + + for (const [index, path] of paths.entries()) { + agent + .get(ORIGIN) + .intercept({ path, method: 'GET' }) + .reply( + 200, + encodeFrame( + { + eventId: `evnt_${index + 1}`, + runId: 'wrun_1', + eventType: 'run_created', + createdAt: CREATED_AT, + eventData: { + deploymentId: 'dpl_1', + workflowName: 'workflow', + input: null, + }, + }, + new Uint8Array() + ), + { headers: { 'content-type': V4_FRAME_CONTENT_TYPE } } + ); + } + + await expect( + getWorkflowRunEventsV4( + { runId: 'wrun_1' }, + { token: 'test-token', dispatcher: agent } + ) + ).rejects.toThrow(/end-of-stream sentinel/); + agent.assertNoPendingInterceptors(); + }); + + it('preserves a clean event-ceiling response', async () => { + const agent = mockAgent(); + agent + .get(ORIGIN) + .intercept({ + path: '/api/v4/runs/wrun_1/events?remoteRefBehavior=resolve&returnAll=true', + method: 'GET', + }) + .reply( + 200, + encodeFrame( + { _end: 1, next: 'eid:evnt_1', hasMore: true }, + new Uint8Array() + ), + { headers: { 'content-type': V4_FRAME_CONTENT_TYPE } } + ); + + const result = await getWorkflowRunEventsV4( + { runId: 'wrun_1' }, + { token: 'test-token', dispatcher: agent } + ); + + expect(result).toEqual({ + data: [], + cursor: 'eid:evnt_1', + hasMore: true, + }); + agent.assertNoPendingInterceptors(); + }); }); /** @@ -589,9 +668,11 @@ describe('getEventsByCorrelationIdV4 over HTTP', () => { }); const result = await getEventsByCorrelationIdV4( - 'step_001', - 'wrun_1', - { limit: 10 }, + { + correlationId: 'step_001', + runId: 'wrun_1', + pagination: { limit: 10 }, + }, { token: 'test-token', dispatcher: agent } ); @@ -603,8 +684,8 @@ describe('getEventsByCorrelationIdV4 over HTTP', () => { expect(query.get('limit')).toBe('10'); } - expect(result.events).toHaveLength(1); - expect(result.events[0].runId).toBe('wrun_1'); + expect(result.data).toHaveLength(1); + expect(result.data[0].runId).toBe('wrun_1'); agent.assertNoPendingInterceptors(); }); }); @@ -686,7 +767,7 @@ describe('v4 transport uses global fetch (observability)', () => { agent .get(origin) .intercept({ - path: '/api/v4/runs/wrun_1/events?returnAll=true', + path: '/api/v4/runs/wrun_1/events?remoteRefBehavior=resolve&returnAll=true', method: 'GET', }) .reply(200, encodeFrame({ _end: 1, hasMore: false }, new Uint8Array(0)), { @@ -698,14 +779,14 @@ describe('v4 transport uses global fetch (observability)', () => { const fetchSpy = vi.spyOn(globalThis, 'fetch'); await getWorkflowRunEventsV4( - 'wrun_1', - {}, + { runId: 'wrun_1' }, { token: 'test-token', dispatcher: agent } ); expect(fetchSpy).toHaveBeenCalledTimes(1); const [calledUrl, calledInit] = fetchSpy.mock.calls[0]; expect(String(calledUrl)).toContain('/api/v4/runs/wrun_1/events'); + expect(calledInit?.signal).toBeUndefined(); agent.assertNoPendingInterceptors(); // Cache-busting header must be set so Next.js fetch memoization / Data @@ -714,6 +795,53 @@ describe('v4 transport uses global fetch (observability)', () => { const sentHeaders = new Headers(calledInit?.headers as HeadersInit); expect(sentHeaders.get('x-request-time')).toBeTruthy(); }); + + it('keeps the normal request deadline on a materialized POST', async () => { + const agent = mockAgent(); + agent + .get(ORIGIN) + .intercept({ + path: '/api/v4/runs/wrun_1/events/step_completed', + method: 'POST', + }) + .reply( + 200, + createEventBody( + { + eventType: 'step_completed', + specVersion: 2, + correlationId: 'step_1', + eventData: { result: new Uint8Array() }, + }, + { + step: { + runId: 'wrun_1', + stepId: 'step_1', + stepName: 'step', + status: 'completed', + attempt: 1, + createdAt: CREATED_AT, + updatedAt: CREATED_AT, + }, + } + ) + ); + const fetchSpy = vi.spyOn(globalThis, 'fetch'); + + await createWorkflowRunEventV4( + { + runId: 'wrun_1', + eventType: 'step_completed', + specVersion: 2, + correlationId: 'step_1', + }, + { token: 'test-token', dispatcher: agent } + ); + + const [, calledInit] = fetchSpy.mock.calls[0]; + expect(calledInit?.signal).toBeInstanceOf(AbortSignal); + agent.assertNoPendingInterceptors(); + }); }); describe('createWorkflowRunEventV4 over HTTP', () => { @@ -891,6 +1019,275 @@ describe('createWorkflowRunEventV4 over HTTP', () => { agent.assertNoPendingInterceptors(); }); + it('continues a truncated run_started stream after its last event', async () => { + const agent = mockAgent(); + + agent + .get(ORIGIN) + .intercept({ + path: '/api/v4/runs/wrun_1/events/run_started', + method: 'POST', + headers: { accept: V4_FRAME_CONTENT_TYPE }, + }) + .reply( + 200, + encodeFrame( + { + eventId: 'evnt_1', + runId: 'wrun_1', + eventType: 'run_created', + createdAt: CREATED_AT, + eventData: { + deploymentId: 'dpl_1', + workflowName: 'workflow', + input: null, + }, + }, + new Uint8Array() + ), + { + headers: { + 'content-type': V4_FRAME_CONTENT_TYPE, + 'x-wf-max-events': '10000', + }, + } + ); + agent + .get(ORIGIN) + .intercept({ + path: '/api/v4/runs/wrun_1/events?cursor=eid%3Aevnt_1&remoteRefBehavior=resolve&returnAll=true', + method: 'GET', + }) + .reply( + 200, + Buffer.concat([ + encodeFrame( + { + eventId: 'evnt_2', + runId: 'wrun_1', + eventType: 'run_started', + createdAt: CREATED_AT, + }, + new Uint8Array() + ), + encodeFrame( + { _end: 1, next: 'eid:evnt_2', hasMore: false }, + new Uint8Array() + ), + ]), + { headers: { 'content-type': V4_FRAME_CONTENT_TYPE } } + ); + + const result = await createWorkflowRunStartedEventV4( + { runId: 'wrun_1', specVersion: 5 }, + { token: 'test-token', dispatcher: agent } + ); + + expect(result.events.map((event) => event.eventId)).toEqual([ + 'evnt_1', + 'evnt_2', + ]); + expect(result.hasMore).toBe(false); + agent.assertNoPendingInterceptors(); + }); + + it('preserves the POST cursor when truncation recovery returns an empty suffix', async () => { + const agent = mockAgent(); + + agent + .get(ORIGIN) + .intercept({ + path: '/api/v4/runs/wrun_1/events/run_started', + method: 'POST', + headers: { accept: V4_FRAME_CONTENT_TYPE }, + }) + .reply( + 200, + Buffer.concat([ + encodeFrame( + { + eventId: 'evnt_1', + runId: 'wrun_1', + eventType: 'run_created', + createdAt: CREATED_AT, + eventData: { + deploymentId: 'dpl_1', + workflowName: 'workflow', + input: null, + }, + }, + new Uint8Array() + ), + encodeFrame( + { + eventId: 'evnt_2', + runId: 'wrun_1', + eventType: 'run_started', + createdAt: CREATED_AT, + }, + new Uint8Array() + ), + ]), + { + headers: { + 'content-type': V4_FRAME_CONTENT_TYPE, + 'x-wf-max-events': '10000', + }, + } + ); + agent + .get(ORIGIN) + .intercept({ + path: '/api/v4/runs/wrun_1/events?cursor=eid%3Aevnt_2&remoteRefBehavior=resolve&returnAll=true', + method: 'GET', + }) + .reply(200, encodeFrame({ _end: 1, hasMore: false }, new Uint8Array()), { + headers: { 'content-type': V4_FRAME_CONTENT_TYPE }, + }); + + const result = await createWorkflowRunStartedEventV4( + { runId: 'wrun_1', specVersion: 5 }, + { token: 'test-token', dispatcher: agent } + ); + + expect(result.events.map((event) => event.eventId)).toEqual([ + 'evnt_1', + 'evnt_2', + ]); + expect(result.cursor).toBe('eid:evnt_2'); + expect(result.hasMore).toBe(false); + agent.assertNoPendingInterceptors(); + }); + + it('retries a truncated continuation that produced no complete event', async () => { + const agent = mockAgent(); + const continuationPath = + '/api/v4/runs/wrun_1/events?cursor=eid%3Aevnt_1&remoteRefBehavior=resolve&returnAll=true'; + + agent + .get(ORIGIN) + .intercept({ + path: '/api/v4/runs/wrun_1/events/run_started', + method: 'POST', + headers: { accept: V4_FRAME_CONTENT_TYPE }, + }) + .reply( + 200, + encodeFrame( + { + eventId: 'evnt_1', + runId: 'wrun_1', + eventType: 'run_created', + createdAt: CREATED_AT, + eventData: { + deploymentId: 'dpl_1', + workflowName: 'workflow', + input: null, + }, + }, + new Uint8Array() + ), + { + headers: { + 'content-type': V4_FRAME_CONTENT_TYPE, + 'x-wf-max-events': '10000', + }, + } + ); + + const runStartedFrame = encodeFrame( + { + eventId: 'evnt_2', + runId: 'wrun_1', + eventType: 'run_started', + createdAt: CREATED_AT, + }, + new Uint8Array() + ); + agent + .get(ORIGIN) + .intercept({ path: continuationPath, method: 'GET' }) + .reply(200, runStartedFrame.subarray(0, 4), { + headers: { 'content-type': V4_FRAME_CONTENT_TYPE }, + }); + agent + .get(ORIGIN) + .intercept({ path: continuationPath, method: 'GET' }) + .reply( + 200, + Buffer.concat([ + runStartedFrame, + encodeFrame( + { _end: 1, next: 'eid:evnt_2', hasMore: false }, + new Uint8Array() + ), + ]), + { headers: { 'content-type': V4_FRAME_CONTENT_TYPE } } + ); + + const result = await createWorkflowRunStartedEventV4( + { runId: 'wrun_1', specVersion: 5 }, + { token: 'test-token', dispatcher: agent } + ); + + expect(result.events.map((event) => event.eventId)).toEqual([ + 'evnt_1', + 'evnt_2', + ]); + expect(result.cursor).toBe('eid:evnt_2'); + agent.assertNoPendingInterceptors(); + }); + + it('preserves a clean event-ceiling run_started response', async () => { + const agent = mockAgent(); + + agent + .get(ORIGIN) + .intercept({ + path: '/api/v4/runs/wrun_1/events/run_started', + method: 'POST', + headers: { accept: V4_FRAME_CONTENT_TYPE }, + }) + .reply( + 200, + Buffer.concat([ + encodeFrame( + { + eventId: 'evnt_1', + runId: 'wrun_1', + eventType: 'run_created', + createdAt: CREATED_AT, + eventData: { + deploymentId: 'dpl_1', + workflowName: 'workflow', + input: null, + }, + }, + new Uint8Array() + ), + encodeFrame( + { _end: 1, next: 'eid:evnt_1', hasMore: true }, + new Uint8Array() + ), + ]), + { + headers: { + 'content-type': V4_FRAME_CONTENT_TYPE, + 'x-wf-max-events': '10000', + }, + } + ); + const result = await createWorkflowRunStartedEventV4( + { runId: 'wrun_1', specVersion: 5 }, + { token: 'test-token', dispatcher: agent } + ); + + expect(result.events.map((event) => event.eventId)).toEqual(['evnt_1']); + expect(result.cursor).toBe('eid:evnt_1'); + expect(result.hasMore).toBe(true); + agent.assertNoPendingInterceptors(); + }); + it('requires the event-stream response requested by run_started', async () => { const origin = WORKFLOW_SERVER_URL_OVERRIDE || 'https://vercel-workflow.com'; @@ -1501,10 +1898,9 @@ describe('v4 POST frame meta forwards every field the splitter produces', () => /** * The recycler in http-client only sees transport failures the v4 client - * reports to it. This covers that wiring end to end: a `fetch()` that rejects - * the way a wedged HTTP/2 session does must retire the shared events pool once - * the failures reach the threshold. Without the `onTransportOutcome` hook in - * `fetchV4` the recycler is never told anything and the pool lives forever. + * reports to it. A streamed response resolves `fetch()` as soon as headers + * arrive, before its body can fail, so the body consumer must own the success + * report or that early success erases every later stream failure. */ describe('v4 transport reports failures to the events recycler', () => { // There is only an undici pool to retire while the adapter owns one: @@ -1528,23 +1924,28 @@ describe('v4 transport reports failures to the events recycler', () => { }), }); - it('rebuilds the shared pool after repeated stream timeouts', async () => { - vi.spyOn(globalThis, 'fetch').mockRejectedValue(wedgedSessionError()); + it('rebuilds the shared pool after repeated response-body timeouts', async () => { + vi.spyOn(globalThis, 'fetch').mockImplementation(async () => { + const body = new ReadableStream({ + start(controller) { + controller.error(wedgedSessionError()); + }, + }); + return new Response(body, { + headers: { 'content-type': V4_FRAME_CONTENT_TYPE }, + }); + }); // No `dispatcher` in the config: the request must resolve the shared one, // which is what the recycler owns. const before = getEventsDispatcher({ token: 'test-token' }); - for (let i = 0; i < EVENTS_RECYCLE_AFTER_CONSECUTIVE_FAILURES; i++) { - await expect( - getWorkflowRunEventsV4('wrun_1', {}, { token: 'test-token' }) - ).rejects.toThrow(); - // Still the same pool until the threshold is reached. - if (i < EVENTS_RECYCLE_AFTER_CONSECUTIVE_FAILURES - 1) { - expect(getEventsDispatcher({ token: 'test-token' })).toBe(before); - } - } + await expect( + getWorkflowRunEventsV4({ runId: 'wrun_1' }, { token: 'test-token' }) + ).rejects.toThrow(); + // The bounded GET retry loop produces enough consecutive body failures to + // retire the wedged pool within this one logical read. expect(getEventsDispatcher({ token: 'test-token' })).not.toBe(before); }); }); diff --git a/packages/world-vercel/src/events-v4.ts b/packages/world-vercel/src/events-v4.ts index 2b9f8f0fe2..2fc003b5d5 100644 --- a/packages/world-vercel/src/events-v4.ts +++ b/packages/world-vercel/src/events-v4.ts @@ -31,21 +31,29 @@ import { EventTypeSchema, getEventDataPayloadField, HookSchema, - type PaginationOptions, + type ListEventsByCorrelationIdParams, + type ListEventsParams, + type PaginatedResponse, StructuredErrorSchema, WaitSchema, WorkflowRunSchema, } from '@workflow/world'; import { decode } from 'cbor-x'; import { z } from 'zod'; +import { + EventPostResponseError, + isRetryableEventRequestError, +} from './event-retry.js'; import { type DecodedFrame, decodeFrames, encodeFrame, + IncompleteFrameError, V4_FRAME_CONTENT_TYPE, } from './frames.js'; import { getEventsDispatcher, + isRecyclableTransportError, noteEventsTransportOutcome, } from './http-client.js'; import { @@ -94,32 +102,39 @@ import { isWsEventsTransportEnabled } from './ws-transport-enabled.js'; * stays on HTTP/1.1 because H2 deadlocks the queue's webhook respondWith * mechanism — see http-client.ts. * - * No per-request timeout: a LIST response streams the full event-log page, which - * for a large run can legitimately take a while to drain — a whole-request - * deadline would abort it mid-stream. + * Event streams opt out of the whole-request timeout because a large replay + * page can legitimately take a while to drain. Materialized single and batch + * writes keep the normal request deadline. */ -async function fetchV4( - url: string, - init: { method: string; headers: Headers; body?: Uint8Array }, - config: APIConfig | undefined, - opName: string, - attributes?: Record +interface V4Request { + url: string; + init: { method: string; headers: Headers; body?: Uint8Array }; + config?: APIConfig; + opName: string; + attributes?: Record; +} + +function instrumentedV4Fetch( + request: V4Request, + options?: { + dispatcher?: ReturnType; + deferTransportSuccess?: boolean; + onTransportOutcome?: (error?: unknown) => void; + } ): Promise { - const dispatcher = getEventsDispatcher(config); + const { url, init, config, opName, attributes } = request; return instrumentedFetch({ - method: init.method, + ...init, url, - headers: init.headers, - body: init.body, - dispatcher, + dispatcher: options?.dispatcher ?? getEventsDispatcher(config), attributes, // Repeated transport failures retire the shared events pool and the next // request builds a fresh one. undici keeps a black-holed HTTP/2 session in // service indefinitely, so without this every request routed onto it fails // until the compute instance is recycled — see noteEventsTransportOutcome. - onTransportOutcome: (error) => - noteEventsTransportOutcome(dispatcher, error), - timeoutMs: null, + onTransportOutcome: options?.onTransportOutcome, + deferTransportSuccess: options?.deferTransportSuccess, + timeoutMs: options?.deferTransportSuccess ? null : undefined, logLabel: opName, // Read the body as bytes, not text: a CBOR error body (the fence 412 // carries event payloads back) does not survive a UTF-8 decode. @@ -134,6 +149,53 @@ async function fetchV4( }); } +/** Materialized responses are transport-complete once their headers arrive. */ +function fetchV4(request: V4Request): Promise { + const dispatcher = getEventsDispatcher(request.config); + return instrumentedV4Fetch(request, { + dispatcher, + onTransportOutcome: (error) => + noteEventsTransportOutcome(dispatcher, error), + }); +} + +/** + * Keep transport accounting scoped to the full response body. Protocol errors + * prove that the origin answered and therefore count as transport success; + * incomplete bodies preserve/report the transport failure instead. + */ +async function withV4ResponseBody( + request: V4Request, + consume: (response: Response) => Promise +): Promise { + const dispatcher = getEventsDispatcher(request.config); + let outcomeReported = false; + const report = (error?: unknown) => { + outcomeReported = true; + noteEventsTransportOutcome(dispatcher, error); + }; + + try { + const response = await instrumentedV4Fetch(request, { + dispatcher, + deferTransportSuccess: true, + onTransportOutcome: report, + }); + const result = await consume(response); + report(); + return result; + } catch (error) { + if (!outcomeReported) { + const incomplete = + error instanceof IncompleteFrameError || + error instanceof PartialEventStreamError || + isRecyclableTransportError(error); + report(incomplete ? error : undefined); + } + throw error; + } +} + const EVENT_ID_HEADER = 'x-wf-event-id'; const MAX_EVENTS_HEADER = 'x-wf-max-events'; @@ -684,18 +746,18 @@ export function throwForErrorResponse( * The frame meta's `eventType` remains authoritative — the backend * cross-checks the two and logs (but does not reject) a mismatch. */ -async function postWorkflowRunEventV4( +async function workflowRunEventV4Request( input: CreateEventV4InputBase & { eventType: EventType; skipPreload?: true; }, - responseType: 'materialized' | 'event-stream', + eventStream: boolean, config?: APIConfig -) { +): Promise { const { baseUrl, headers: baseHeaders } = await getHttpConfig(config); const headers = new Headers(baseHeaders); headers.set('Content-Type', 'application/octet-stream'); - if (responseType === 'event-stream') { + if (eventStream) { headers.set('Accept', V4_FRAME_CONTENT_TYPE); } @@ -705,12 +767,12 @@ async function postWorkflowRunEventV4( ); const url = eventsV4Url(baseUrl, input.runId, input.eventType); - return fetchV4( + return { url, - { method: 'POST', headers, body: frame }, + init: { method: 'POST', headers, body: frame }, config, - 'createEvent', - { + opName: 'createEvent', + attributes: { ...WorkflowEventsTransport('http'), ...WorkflowEventType(input.eventType), ...WorkflowClientVersion(`@workflow/world-vercel/${version}`), @@ -718,7 +780,31 @@ async function postWorkflowRunEventV4( ...(input.optimizations !== undefined ? StepLatencyOptimizations(input.optimizations) : {}), - } + }, + }; +} + +async function postWorkflowRunEventV4( + input: CreateEventV4InputBase & { + eventType: EventType; + skipPreload?: true; + }, + config?: APIConfig +): Promise { + return fetchV4(await workflowRunEventV4Request(input, false, config)); +} + +async function withWorkflowRunEventResponseBody( + input: CreateEventV4InputBase & { + eventType: EventType; + skipPreload?: true; + }, + config: APIConfig | undefined, + consume: (response: Response) => Promise +): Promise { + return withV4ResponseBody( + await workflowRunEventV4Request(input, true, config), + consume ); } @@ -739,7 +825,7 @@ export async function createWorkflowRunEventV4( if (reply) return decodeCreateEventResponse(reply, input.eventType); } - const response = await postWorkflowRunEventV4(input, 'materialized', config); + const response = await postWorkflowRunEventV4(input, config); const contentType = response.headers.get('content-type'); if (contentType?.startsWith(V4_FRAME_CONTENT_TYPE)) { @@ -781,16 +867,13 @@ export async function createWorkflowRunStartedEventV4( input: CreateEventV4InputBase, config?: APIConfig ) { - const response = await postWorkflowRunEventV4( + const { responseHeaders, ...replay } = await postReplayLogEvent( { ...input, eventType: 'run_started' }, - 'event-stream', config ); - const events: Event[] = []; - const page = await consumeEventFrameStream(response, 'createEvent', events); - assert(page.cursor, 'v4 createEvent: event stream missing cursor'); + assert(replay.cursor, 'v4 createEvent: event stream missing cursor'); const maxEvents = MaxEventsHeaderSchema.safeParse( - response.headers.get(MAX_EVENTS_HEADER) + responseHeaders.get(MAX_EVENTS_HEADER) ); if (!maxEvents.success) { throw new WorkflowWorldError('v4 createEvent: invalid max-events header', { @@ -799,7 +882,7 @@ export async function createWorkflowRunStartedEventV4( }); } - return { events, ...page, maxEvents: maxEvents.data }; + return { ...replay, maxEvents: maxEvents.data }; } /** One event of a v4 batch POST, index-aligned with the response results. */ @@ -887,16 +970,16 @@ export async function createWorkflowRunEventsBatchV4( // span carries only wire-level facts. workflow.event.type is deliberately // absent — it names a single event write, and tagging a batch with its // first event's type misclassifies the traffic. - const response = await fetchV4( + const response = await fetchV4({ url, - { method: 'POST', headers, body }, + init: { method: 'POST', headers, body }, config, - 'createEventBatch', - { + opName: 'createEventBatch', + attributes: { ...WorkflowEventsTransport('http'), 'workflow.batch.bytes': body.byteLength, - } - ); + }, + }); const bodyBytes = new Uint8Array(await response.arrayBuffer()); const decoded = @@ -1183,73 +1266,41 @@ async function postEventFrameOverWs( ); } -/** - * Result of a `hook_received` POST that opted into the replay-log preload, - * discriminated on `kind` (keyed on the response content type). - */ -export type HookReceivedPreloadV4Result = - /** The server streamed the replay log back as v4 frames. */ - | (ListEventsV4Result & { - kind: 'stream'; - /** - * The canonical event this write created or converged on (the resume - * claim winner's — ours or the producer's), named by the - * event-id response header. Undefined when the server did not send it. - */ - canonicalEventId: string | undefined; - /** Per-run event ceiling from the response header, when present. */ - maxEvents: number | undefined; - }) - /** - * The server answered with the normal materialized CBOR body instead — - * an older server, or one that declined the optimization. The - * hook_received write itself has still succeeded; callers must not - * re-post it. - */ - | { - kind: 'materialized'; - result: EventResult<'hook_received'> & { event: Event }; - }; +interface ReplayLog { + events: Event[]; + cursor: string | null; + hasMore: boolean; +} /** * POST /api/v4/runs/:runId/events/hook_received with the v4-frame `Accept`, - * consuming either response mode. + * consuming the replay-log response. * - * A server that supports the lazy-hook replay stream answers the consumer's - * idempotent re-ensure with the run's complete replay log as v4 frames — - * the same event-frame sequence LIST uses, ending with the `_end` sentinel. - * A truncated stream (EOF without the sentinel) throws; the write is - * deduplicated by the server's `(runId, resumeId)` constraint, so retrying - * the whole request is safe and converges on the same canonical event. + * The consumer's idempotent re-ensure always answers with the run's replay log + * as v4 frames. A truncated stream resumes with a GET after its last validated + * event. If it fails before producing any event, the outer POST retry remains + * safe because the server deduplicates `(runId, resumeId)` and returns the + * canonical event. */ export async function createHookReceivedPreloadEventV4( input: CreateEventV4InputBase, config?: APIConfig -): Promise { - const response = await postWorkflowRunEventV4( +): Promise< + ReplayLog & { + canonicalEventId: string | undefined; + maxEvents: number | undefined; + } +> { + const { responseHeaders, ...replay } = await postReplayLogEvent( { ...input, eventType: 'hook_received' }, - 'event-stream', config ); - - const contentType = response.headers.get('content-type'); - if (!contentType?.startsWith(V4_FRAME_CONTENT_TYPE)) { - return { - kind: 'materialized', - result: await decodeCreateEventResponse(response, 'hook_received'), - }; - } - - const events: Event[] = []; - const page = await consumeEventFrameStream(response, 'createEvent', events); const maxEvents = MaxEventsHeaderSchema.safeParse( - response.headers.get(MAX_EVENTS_HEADER) + responseHeaders.get(MAX_EVENTS_HEADER) ); return { - kind: 'stream', - events, - ...page, - canonicalEventId: response.headers.get(EVENT_ID_HEADER) ?? undefined, + ...replay, + canonicalEventId: responseHeaders.get(EVENT_ID_HEADER) ?? undefined, maxEvents: maxEvents.success ? maxEvents.data : undefined, }; } @@ -1281,108 +1332,137 @@ export async function getEventV4( const url = `${baseUrl}/v4/runs/${encodeURIComponent(runId)}/events/${encodeURIComponent(eventId)}` + `?remoteRefBehavior=${remoteRefBehavior}`; - const response = await fetchV4( - url, - { method: 'GET', headers }, - config, - 'getEvent' + return withV4ResponseBody( + { + url, + init: { method: 'GET', headers }, + config, + opName: 'getEvent', + }, + async (response) => { + // GET emits one frame without a sentinel. + for await (const frame of decodeFrames( + eventFrameChunks(response, 'getEvent') + )) { + return decodeEventFrame(frame); + } + throw new IncompleteFrameError( + `v4 getEvent: empty frame stream for ${eventId}` + ); + } ); +} + +function eventFrameChunks( + response: Response, + opName: string +): AsyncIterable { const contentType = response.headers.get('content-type'); if (!contentType?.startsWith(V4_FRAME_CONTENT_TYPE)) { throw new Error( - `v4 getEvent: expected ${V4_FRAME_CONTENT_TYPE}, got ${contentType ?? '(none)'}` + `v4 ${opName}: expected ${V4_FRAME_CONTENT_TYPE}, got ${contentType ?? '(none)'}` ); } + if (!response.body) { + throw new IncompleteFrameError(`v4 ${opName}: response body is missing`); + } // fetch's `Response.body` is a web ReadableStream, which is async-iterable - // on Node (readableStream async iteration, since v16.5.0) — feed it straight - // to decodeFrames. The cast is only because TS's lib `ReadableStream` type - // omits the async iterator. Do NOT round-trip through `node:stream` - // Readable.toWeb: a dynamic `import('node:stream')` resolves to an empty - // module namespace in Next.js webpack server bundles and crashes. - const chunks = response.body as unknown as AsyncIterable; - - // GET emits a single frame (no sentinel); decodeFrames returns at EOF - // after yielding it. - for await (const frame of decodeFrames(chunks)) { - return decodeEventFrame(frame); - } - throw new Error(`v4 getEvent: empty frame stream for ${eventId}`); + // on Node. The cast is only because TS's lib `ReadableStream` type omits the + // async iterator. A node:stream conversion breaks in Next.js webpack server + // bundles, where that dynamic import resolves to an empty namespace. + return response.body as unknown as AsyncIterable; } -export interface ListEventsV4Params extends PaginationOptions { - /** - * Whether the backend resolves payload bytes into each frame body. - * `resolve` (default) streams the bytes; `lazy` emits empty-body frames - * (the ref descriptor stays in the frame meta) — for metadata-only - * listings that would otherwise download every payload just to discard - * it. - */ - remoteRefBehavior?: 'resolve' | 'lazy'; +class PartialEventStreamError extends WorkflowWorldError { + constructor(message: string, cause?: unknown) { + super(message, { code: 'TRANSPORT', cause }); + } } -export interface ListEventsV4Result { - events: Event[]; - /** Trailing event-log cursor, or null when the stream contained no events. */ - cursor: string | null; - /** Explicit "another page of results exists" flag from the sentinel. */ - hasMore: boolean; -} +const MAX_PARTIAL_EVENT_STREAM_RETRIES = 3; async function consumeEventFrameStream( response: Response, opName: string, events: Event[] -): Promise> { - const contentType = response.headers.get('content-type'); - if (!contentType?.startsWith(V4_FRAME_CONTENT_TYPE)) { - throw new Error( - `v4 ${opName}: expected ${V4_FRAME_CONTENT_TYPE}, got ${contentType ?? '(none)'}` - ); - } - - const chunks = response.body as unknown as AsyncIterable; - - for await (const frame of decodeFrames(chunks)) { - if (frame.meta._end === 1) { - const end = EventStreamEndSchema.parse(frame.meta); - return { cursor: end.next ?? null, hasMore: end.hasMore }; - } - if (Object.keys(frame.meta).some((key) => key.startsWith('_'))) { - throw new Error(`v4 ${opName}: unexpected control frame`); +): Promise<{ cursor: string | null; hasMore: boolean }> { + try { + for await (const frame of decodeFrames( + eventFrameChunks(response, opName) + )) { + if (frame.meta._end === 1) { + const end = EventStreamEndSchema.parse(frame.meta); + return { + cursor: end.next ?? null, + hasMore: end.hasMore, + }; + } + if (Object.keys(frame.meta).some((key) => key.startsWith('_'))) { + throw new Error(`v4 ${opName}: unexpected control frame`); + } + events.push(decodeEventFrame(frame)); } - events.push(decodeEventFrame(frame)); + } catch (cause) { + if (!(cause instanceof IncompleteFrameError)) throw cause; + throw new PartialEventStreamError( + `v4 ${opName}: event frame stream failed after ${events.length} events`, + cause + ); } - throw new Error( + throw new PartialEventStreamError( `v4 ${opName}: frame stream ended without the end-of-stream sentinel ` + `(${events.length} events read) — truncated response?` ); } -/** - * Drive a v4 frame-stream list response into an in-memory page. Used by - * both the by-runId and by-correlationId list endpoints — the wire - * shape is identical, only the URL differs. - * - * `headers` come from the caller's single getHttpConfig resolution (the - * same call that produced the baseUrl in `url`) so each LIST resolves - * auth exactly once. - */ -async function consumeListFrameStream( - url: string, - headers: Headers, - config: APIConfig | undefined, - opName: string, - events: Event[] -): Promise> { - const response = await fetchV4( - url, - { method: 'GET', headers }, - config, - opName - ); - return consumeEventFrameStream(response, opName, events); +/** POST a replay log and continue only an actually incomplete response body. */ +async function postReplayLogEvent( + input: CreateEventV4InputBase & { + eventType: 'run_started' | 'hook_received'; + }, + config?: APIConfig +): Promise { + const events: Event[] = []; + let responseHeaders: Headers | undefined; + try { + const page = await withWorkflowRunEventResponseBody( + input, + config, + (response) => { + responseHeaders = response.headers; + return consumeEventFrameStream(response, 'createEvent', events); + } + ); + assert(responseHeaders); + return { events, ...page, responseHeaders }; + } catch (error) { + if (!(error instanceof PartialEventStreamError)) throw error; + const lastEvent = events.at(-1); + if (!lastEvent) throw error; + assert(responseHeaders); + const continuationCursor = `eid:${lastEvent.eventId}`; + + try { + const suffix = await getWorkflowRunEventsV4( + { runId: input.runId, pagination: { cursor: continuationCursor } }, + config + ); + events.push(...suffix.data); + return { + events, + cursor: suffix.cursor ?? continuationCursor, + hasMore: suffix.hasMore, + responseHeaders, + }; + } catch (cause) { + throw new EventPostResponseError( + `v4 createEvent: replay continuation failed for run ${input.runId}`, + { cause } + ); + } + } } /** @@ -1390,20 +1470,29 @@ async function consumeListFrameStream( * Shared by the runId and correlationId list query builders so both send * `remoteRefBehavior` identically. */ -function appendListParams(sp: URLSearchParams, params: ListEventsV4Params) { - if (params.cursor) sp.set('cursor', params.cursor); - if (params.limit !== undefined) sp.set('limit', String(params.limit)); - if (params.sortOrder) sp.set('sortOrder', params.sortOrder); - if (params.remoteRefBehavior) { - sp.set('remoteRefBehavior', params.remoteRefBehavior); - } +function appendListParams( + sp: URLSearchParams, + params: ListEventsParams | ListEventsByCorrelationIdParams, + cursor: string | null +) { + const { limit, sortOrder } = params.pagination ?? {}; + if (cursor) sp.set('cursor', cursor); + if (limit !== undefined) sp.set('limit', String(limit)); + if (sortOrder) sp.set('sortOrder', sortOrder); + sp.set( + 'remoteRefBehavior', + params.resolveData === 'none' ? 'lazy' : 'resolve' + ); } -function paginationToQuery(params: ListEventsV4Params): string { +function paginationToQuery( + params: ListEventsParams, + cursor: string | null +): string { const sp = new URLSearchParams(); // The World API uses an omitted limit for a complete event log. - if (params.limit === undefined) sp.set('returnAll', 'true'); - appendListParams(sp, params); + if (params.pagination?.limit === undefined) sp.set('returnAll', 'true'); + appendListParams(sp, params, cursor); return `?${sp.toString()}`; } @@ -1418,37 +1507,38 @@ function paginationToQuery(params: ListEventsV4Params): string { * after its last validated event instead of downloading accepted frames again. */ export async function getWorkflowRunEventsV4( - runId: string, - params: ListEventsV4Params = {}, + params: ListEventsParams, config?: APIConfig -): Promise { +): Promise> { const { baseUrl, headers } = await getHttpConfig(config); const events: Event[] = []; - let cursor = params.cursor; + let cursor = params.pagination?.cursor ?? null; - while (true) { + for (let retries = 0; ; retries++) { const url = - `${baseUrl}/v4/runs/${encodeURIComponent(runId)}/events` + - paginationToQuery({ ...params, cursor }); + `${baseUrl}/v4/runs/${encodeURIComponent(params.runId)}/events` + + paginationToQuery(params, cursor); try { - const page = await consumeListFrameStream( - url, - headers, - config, - 'listEvents', - events + const result = await withV4ResponseBody( + { + url, + init: { method: 'GET', headers }, + config, + opName: 'listEvents', + }, + (response) => consumeEventFrameStream(response, 'listEvents', events) ); - return { events, ...page }; + return { data: events, cursor: result.cursor, hasMore: result.hasMore }; } catch (error) { const lastEvent = events.at(-1); if ( - params.limit !== undefined || - !lastEvent || - `eid:${lastEvent.eventId}` === cursor + retries === MAX_PARTIAL_EVENT_STREAM_RETRIES || + params.pagination?.limit !== undefined || + !isRetryableEventRequestError(error) ) { throw error; } - cursor = `eid:${lastEvent.eventId}`; + if (lastEvent) cursor = `eid:${lastEvent.eventId}`; } } } @@ -1468,24 +1558,25 @@ export async function getWorkflowRunEventsV4( * the page by run id. */ export async function getEventsByCorrelationIdV4( - correlationId: string, - runId: string, - params: ListEventsV4Params = {}, + params: ListEventsByCorrelationIdParams, config?: APIConfig -): Promise { +): Promise> { const { baseUrl, headers } = await getHttpConfig(config); const sp = new URLSearchParams(); - sp.set('correlationId', correlationId); - sp.set('runId', runId); - appendListParams(sp, params); + sp.set('correlationId', params.correlationId); + sp.set('runId', params.runId); + appendListParams(sp, params, params.pagination?.cursor ?? null); const url = `${baseUrl}/v4/events?${sp.toString()}`; const events: Event[] = []; - const page = await consumeListFrameStream( - url, - headers, - config, - 'listEventsByCorrelationId', - events + const result = await withV4ResponseBody( + { + url, + init: { method: 'GET', headers }, + config, + opName: 'listEventsByCorrelationId', + }, + (response) => + consumeEventFrameStream(response, 'listEventsByCorrelationId', events) ); - return { events, ...page }; + return { data: events, cursor: result.cursor, hasMore: result.hasMore }; } diff --git a/packages/world-vercel/src/events.test.ts b/packages/world-vercel/src/events.test.ts index 5fea9675a4..c396ad9724 100644 --- a/packages/world-vercel/src/events.test.ts +++ b/packages/world-vercel/src/events.test.ts @@ -5,9 +5,11 @@ import { decode, encode } from 'cbor-x'; import { ulid } from 'ulid'; import { MockAgent } from 'undici'; import { describe, expect, it } from 'vitest'; +import { EventPostResponseError } from './event-retry.js'; import { createWorkflowRunEvent, getWorkflowRunEvents, + getWorkflowRunEventsByCorrelationId, splitEventDataForV4, } from './events.js'; import { encodeFrame, V4_FRAME_CONTENT_TYPE } from './frames.js'; @@ -373,6 +375,60 @@ describe('createWorkflowRunEvent result contract', () => { ).rejects.toMatchObject(error); agent.assertNoPendingInterceptors(); }); + + it('does not repeat a run_started POST when its continuation exhausts retries', async () => { + const agent = mockAgent(); + agent + .get(ORIGIN) + .intercept({ + path: '/api/v4/runs/wrun_1/events/run_started', + method: 'POST', + }) + .reply( + 200, + encodeFrame( + { + eventId: 'evnt_0', + runId: 'wrun_1', + eventType: 'run_created', + createdAt: new Date('2026-06-09T23:59:59.000Z'), + specVersion: 2, + eventData: { + deploymentId: 'dpl_1', + workflowName: 'workflow', + }, + }, + new Uint8Array() + ), + { + headers: { + 'content-type': V4_FRAME_CONTENT_TYPE, + 'x-wf-max-events': '10000', + }, + } + ); + + const continuationPath = + '/api/v4/runs/wrun_1/events?cursor=eid%3Aevnt_0&remoteRefBehavior=resolve&returnAll=true'; + for (let attempt = 0; attempt < 4; attempt++) { + agent + .get(ORIGIN) + .intercept({ path: continuationPath, method: 'GET' }) + .reply(200, new Uint8Array(), { + headers: { 'content-type': V4_FRAME_CONTENT_TYPE }, + }); + } + + await expect( + createWorkflowRunEvent( + 'wrun_1', + { eventType: 'run_started', specVersion: 2 } as AnyEventRequest, + undefined, + { token: 'test-token', dispatcher: agent } + ) + ).rejects.toBeInstanceOf(EventPostResponseError); + agent.assertNoPendingInterceptors(); + }); }); /** POSTs a v4 step_started with `params` and returns the decoded frame meta. */ @@ -1393,7 +1449,14 @@ describe('getWorkflowRunEvents legacy structured-error compatibility', () => { * fallback preserves their (correct, if slower) behavior. */ describe('getWorkflowRunEvents hasMore mapping', () => { - function mockListResponse(agent: MockAgent, sentinelMeta: object) { + function mockListResponse( + agent: MockAgent, + sentinelMeta: object, + query: Record = { + returnAll: 'true', + remoteRefBehavior: 'resolve', + } + ) { const frames = Buffer.concat([ encodeFrame( { @@ -1412,9 +1475,7 @@ describe('getWorkflowRunEvents hasMore mapping', () => { .intercept({ path: '/api/v4/runs/wrun_1/events', method: 'GET', - // These tests omit the limit and use the default resolveData - // ('all' → resolve); match both translated query params. - query: { returnAll: 'true', remoteRefBehavior: 'resolve' }, + query, }) .reply(200, frames, { headers: { 'content-type': V4_FRAME_CONTENT_TYPE }, @@ -1439,10 +1500,14 @@ describe('getWorkflowRunEvents hasMore mapping', () => { it('maps an explicit hasMore:true through', async () => { const agent = mockAgent(); - mockListResponse(agent, { _end: 1, next: 'cursor-2', hasMore: true }); + mockListResponse( + agent, + { _end: 1, next: 'cursor-2', hasMore: true }, + { limit: '500', remoteRefBehavior: 'resolve' } + ); const result = await getWorkflowRunEvents( - { runId: 'wrun_1' }, + { runId: 'wrun_1', pagination: { limit: 500 } }, { token: 'test-token', dispatcher: agent } ); @@ -1511,7 +1576,7 @@ describe('getWorkflowRunEvents by correlation id is scoped to the run', () => { headers: { 'content-type': V4_FRAME_CONTENT_TYPE }, }); - const result = await getWorkflowRunEvents( + const result = await getWorkflowRunEventsByCorrelationId( { correlationId: 'step_001', runId: 'wrun_1' }, { token: 'test-token', dispatcher: agent } ); @@ -1654,9 +1719,8 @@ describe('createWorkflowRunEvent hook_received replay preload', () => { ); // The idempotency key + digest rode the frame meta. The request keeps - // hook_received's lazy default: a supporting server owns frame-body - // resolution regardless, and an older server then answers the CBOR - // fallback without resolving a payload the runtime would discard. + // hook_received's lazy default; the preload contract makes the server own + // resolution of the replay-ready frame bodies. expect(capturedMeta?.resumeId).toBe(RESUME_ID); expect(capturedMeta?.resumePayloadDigest).toBe(DIGEST); expect(capturedMeta?.remoteRefBehavior).toBe('lazy'); @@ -1795,7 +1859,7 @@ describe('createWorkflowRunEvent hook_received replay preload', () => { agent.assertNoPendingInterceptors(); }); - it('keeps the CBOR result when the server does not stream (older server)', async () => { + it('requires the framed preload response promised by workflow-server', async () => { const agent = mockAgent(); agent .get(ORIGIN) @@ -1804,45 +1868,22 @@ describe('createWorkflowRunEvent hook_received replay preload', () => { method: 'POST', headers: { accept: V4_FRAME_CONTENT_TYPE }, }) - .reply( - 200, - encode({ - event: { - eventId: 'evnt_4', - runId: 'wrun_1', - eventType: 'hook_received', - correlationId: 'hook_1', - createdAt: new Date('2026-06-10T00:00:03.000Z'), - specVersion: 2, - eventData: { token: 'tok-preload' }, - }, - }), - { - headers: { - 'content-type': 'application/cbor', - 'x-wf-event-id': 'evnt_4', - 'x-wf-run-id': 'wrun_1', - 'x-wf-created-at': '2026-06-10T00:00:03.000Z', - }, - } - ); + .reply(200, encode({ event: { eventType: 'hook_received' } }), { + headers: { 'content-type': 'application/cbor' }, + }); - const result = await createWorkflowRunEvent( - 'wrun_1', - hookReceivedRequest(), - preloadParams, - { token: 'test-token', dispatcher: agent } + await expect( + createWorkflowRunEvent('wrun_1', hookReceivedRequest(), preloadParams, { + token: 'test-token', + dispatcher: agent, + }) + ).rejects.toThrow( + `v4 createEvent: expected ${V4_FRAME_CONTENT_TYPE}, got application/cbor` ); - - // A successful write with no replay preload — the runtime falls back to - // the run_started setup without posting the hook again. - expect(result.event?.eventType).toBe('hook_received'); - expect(result.events).toBeUndefined(); - expect(result.run).toBeUndefined(); agent.assertNoPendingInterceptors(); }); - it('rejects a truncated preload stream (no end sentinel)', async () => { + it('continues a truncated preload stream after its last event', async () => { const agent = mockAgent(); agent .get(ORIGIN) @@ -1866,15 +1907,43 @@ describe('createWorkflowRunEvent hook_received replay preload', () => { }, PAYLOAD ), + { + headers: { + 'content-type': V4_FRAME_CONTENT_TYPE, + 'x-wf-event-id': 'evnt_4', + }, + } + ); + + agent + .get(ORIGIN) + .intercept({ + path: '/api/v4/runs/wrun_1/events?cursor=eid%3Aevnt_4&remoteRefBehavior=resolve&returnAll=true', + method: 'GET', + }) + .reply( + 200, + encodeFrame( + { _end: 1, next: 'eid:evnt_4', hasMore: false }, + new Uint8Array() + ), { headers: { 'content-type': V4_FRAME_CONTENT_TYPE } } ); - await expect( - createWorkflowRunEvent('wrun_1', hookReceivedRequest(), preloadParams, { + const result = await createWorkflowRunEvent( + 'wrun_1', + hookReceivedRequest(), + preloadParams, + { token: 'test-token', dispatcher: agent, - }) - ).rejects.toThrow(/end-of-stream sentinel/); + } + ); + + expect(result.event?.eventId).toBe('evnt_4'); + expect(result.events).toHaveLength(1); + expect(result.cursor).toBe('eid:evnt_4'); + expect(result.hasMore).toBe(false); agent.assertNoPendingInterceptors(); }); diff --git a/packages/world-vercel/src/events.ts b/packages/world-vercel/src/events.ts index 9706a59af9..d2903ff116 100644 --- a/packages/world-vercel/src/events.ts +++ b/packages/world-vercel/src/events.ts @@ -31,6 +31,7 @@ * the v3 path. */ +import assert from 'node:assert/strict'; import { HookNotFoundError, WorkflowWorldError } from '@workflow/errors'; import { type AnyEventRequest, @@ -62,15 +63,10 @@ import { getEventsByCorrelationIdV4, getEventV4, getWorkflowRunEventsV4, - type ListEventsV4Params, } from './events-v4.js'; import { decode as decodeRunId } from './run-id/index.js'; import { cancelWorkflowRunV1, createWorkflowRunV1 } from './runs.js'; -import { - type APIConfig, - DEFAULT_RESOLVE_DATA_OPTION, - makeRequest, -} from './utils.js'; +import { type APIConfig, makeRequest } from './utils.js'; function validateWorkflowRunIdTimestamp(id: string): string | null { const raw = id.startsWith('wrun_') ? id.slice('wrun_'.length) : id; @@ -436,26 +432,17 @@ export async function getEvent( } export async function getWorkflowRunEvents( - params: ListEventsParams | ListEventsByCorrelationIdParams, + params: ListEventsParams, config?: APIConfig ): Promise> { - const { pagination, resolveData = DEFAULT_RESOLVE_DATA_OPTION } = params; - // `resolveData: 'none'` leaves payload refs unresolved, so the backend can - // skip reading and streaming their contents. The validated lazy descriptors - // remain on the returned events. - const listParams: ListEventsV4Params = { - ...pagination, - remoteRefBehavior: resolveData === 'none' ? 'lazy' : 'resolve', - }; + return getWorkflowRunEventsV4(params, config); +} - const result = await ('correlationId' in params - ? getEventsByCorrelationIdV4( - params.correlationId, - params.runId, - listParams, - config - ) - : getWorkflowRunEventsV4(params.runId, listParams, config)); +export async function getWorkflowRunEventsByCorrelationId( + params: ListEventsByCorrelationIdParams, + config?: APIConfig +): Promise> { + const result = await getEventsByCorrelationIdV4(params, config); // A correlation id is unique per run, not globally — a slot-numbered run // numbers its own steps, so `step_…001` names the first step of every such @@ -464,10 +451,7 @@ export async function getWorkflowRunEvents( // `hasMore`/`cursor` stay the backend's, so a page that filters down to // nothing is still followed by the next one. return { - data: - 'correlationId' in params - ? result.events.filter((event) => event.runId === params.runId) - : result.events, + data: result.data.filter((event) => event.runId === params.runId), // The cursor is present even on the final page because it is also the // incremental-load resume point. `hasMore` is the pagination signal. cursor: result.cursor, @@ -769,32 +753,12 @@ async function createWorkflowRunEventInner( 'v4 createEvent: run_started stream is missing run_started' ); } - - let attributes = runCreated.eventData.attributes ?? {}; - let updatedAt = runStarted.createdAt; - for (const event of result.events) { - if (event.eventType === 'attr_set') { - attributes = applyAttributeChanges(attributes, event.eventData.changes); - updatedAt = event.createdAt; - } - } + const run = reconstructRunFromReplayEvents(result.events); + assert(run); return { event: runStarted, - run: { - runId: runCreated.runId, - status: 'running', - deploymentId: runCreated.eventData.deploymentId, - workflowName: runCreated.eventData.workflowName, - specVersion: runCreated.specVersion, - executionContext: runCreated.eventData.executionContext, - input: runCreated.eventData.input, - attributes, - encryptionPublicKey: runCreated.eventData.encryptionPublicKey, - startedAt: runStarted.createdAt, - createdAt: runCreated.createdAt, - updatedAt, - }, + run, events: result.events, cursor: result.cursor, hasMore: result.hasMore, @@ -810,23 +774,15 @@ async function createWorkflowRunEventInner( ) { // Lazy hook resume: the queue consumer's idempotent re-ensure doubles // as the invocation's setup request. A supporting server streams the - // complete replay log back in this response with resolved frame bodies - // — the SERVER owns that resolution (the preload contract requires + // complete replay log back in this response with resolved frame bodies. + // The SERVER owns that resolution (the preload contract requires // replay-ready bytes; v4 has no /refs endpoint to hydrate a lazy // descriptor during replay), so the request keeps hook_received's lazy - // default. Against an older server this makes the CBOR fallback - // lightweight: it answers the mutation without resolving and echoing - // an S3-backed hook payload the runtime would discard anyway. + // default. const outcome = await createHookReceivedPreloadEventV4( { ...input, remoteRefBehavior: 'lazy' }, config ); - if (outcome.kind === 'materialized') { - // Older server (or optimization declined): the write still succeeded - // and this is its normal materialized result. The runtime sees no - // replay preload on it and falls back to the run_started setup. - return outcome.result; - } const { canonicalEventId, maxEvents, events, cursor, hasMore } = outcome; const canonicalEvent = events.find( (event) => event.eventId === canonicalEventId diff --git a/packages/world-vercel/src/frames.test.ts b/packages/world-vercel/src/frames.test.ts index 72d312272e..036509052b 100644 --- a/packages/world-vercel/src/frames.test.ts +++ b/packages/world-vercel/src/frames.test.ts @@ -4,6 +4,8 @@ import { type DecodedFrame, decodeFrames, encodeFrame, + IncompleteFrameError, + InvalidFrameError, V4_FRAME_CONTENT_TYPE, } from './frames.js'; @@ -174,7 +176,36 @@ describe('decodeFrames', () => { it('throws when the stream ends mid-frame', async () => { const partial = encodeFrame({ x: 1 }, new Uint8Array(100)).slice(0, 20); const stream = streamOf(partial, 1024); - await expect(drainFrames(stream)).rejects.toThrow(/truncated/); + await expect(drainFrames(stream)).rejects.toBeInstanceOf( + IncompleteFrameError + ); + }); + + it('distinguishes invalid CBOR metadata from an incomplete response', async () => { + const invalidMeta = new Uint8Array([0, 0, 0, 1, 0x01, 0, 0, 0, 0]); + + await expect(drainFrames(streamOf(invalidMeta, 9))).rejects.toBeInstanceOf( + InvalidFrameError + ); + }); + + it('classifies a source read failure as an incomplete response', async () => { + const cause = new Error('socket reset'); + async function* failingSource(): AsyncGenerator { + yield new Uint8Array([0, 0]); + throw cause; + } + + await expect( + (async () => { + for await (const _frame of decodeFrames(failingSource())) { + // drain + } + })() + ).rejects.toMatchObject({ + name: 'IncompleteFrameError', + cause, + }); }); it('preserves CBOR types in meta (numbers, booleans, arrays)', async () => { diff --git a/packages/world-vercel/src/frames.ts b/packages/world-vercel/src/frames.ts index f5de347979..8fdffd976f 100644 --- a/packages/world-vercel/src/frames.ts +++ b/packages/world-vercel/src/frames.ts @@ -18,10 +18,48 @@ export interface DecodedFrame { body: Uint8Array; } +/** The response body stopped before the next complete frame was available. */ +export class IncompleteFrameError extends Error { + constructor(message: string, options?: ErrorOptions) { + super(message, options); + this.name = 'IncompleteFrameError'; + } +} + +/** The body was complete enough to decode, but did not contain valid frame metadata. */ +export class InvalidFrameError extends Error { + constructor(message: string, options?: ErrorOptions) { + super(message, options); + this.name = 'InvalidFrameError'; + } +} + // The protocol consumer validates the event or control-frame shape after the // body is available. The byte codec only requires a CBOR object here. const CborObjectSchema = z.record(z.string(), z.unknown()); +async function readChunk( + chunks: AsyncIterator +): Promise> { + try { + return await chunks.next(); + } catch (cause) { + throw new IncompleteFrameError('decodeFrames: source stream failed', { + cause, + }); + } +} + +function decodeFrameMeta(bytes: Uint8Array): Record { + try { + return CborObjectSchema.parse(decode(bytes)); + } catch (cause) { + throw new InvalidFrameError('decodeFrames: invalid CBOR metadata', { + cause, + }); + } +} + /** Test/utility: encode a complete frame. Production server uses prefix * + streaming body. */ export function encodeFrame( @@ -71,7 +109,7 @@ export async function* decodeFrames( const parts: Uint8Array[] = [buffer]; let byteLength = buffer.byteLength; while (byteLength < needed) { - const chunk = await chunks.next(); + const chunk = await readChunk(chunks); if (chunk.done) return false; if (chunk.value.byteLength === 0) continue; parts.push(chunk.value); @@ -104,12 +142,12 @@ export async function* decodeFrames( take(4); if (!(await refill(metaLen))) { - throw new Error('decodeFrames: truncated meta block'); + throw new IncompleteFrameError('decodeFrames: truncated meta block'); } - const meta = CborObjectSchema.parse(decode(take(metaLen))); + const meta = decodeFrameMeta(take(metaLen)); if (!(await refill(4))) { - throw new Error('decodeFrames: truncated body length'); + throw new IncompleteFrameError('decodeFrames: truncated body length'); } const bodyLen = new DataView( buffer.buffer, @@ -119,7 +157,7 @@ export async function* decodeFrames( take(4); if (bodyLen > 0 && !(await refill(bodyLen))) { - throw new Error('decodeFrames: truncated body bytes'); + throw new IncompleteFrameError('decodeFrames: truncated body bytes'); } // Slice (not subarray) so the yielded body owns its bytes — later // reads into the buffer won't overwrite it; bodyLen 0 yields empty. diff --git a/packages/world-vercel/src/http-core.ts b/packages/world-vercel/src/http-core.ts index b8122bb7fd..cc5c2086c4 100644 --- a/packages/world-vercel/src/http-core.ts +++ b/packages/world-vercel/src/http-core.ts @@ -487,6 +487,8 @@ export interface InstrumentedFetchOptions extends HttpClientSpanOptions { * connections stop delivering (see noteEventsTransportOutcome). */ onTransportOutcome?: (error?: unknown) => void; + /** Let a streaming body consumer report success after it finishes. */ + deferTransportSuccess?: boolean; } /** @@ -520,6 +522,7 @@ export async function instrumentedFetch( attributes, durationAttribute, onTransportOutcome, + deferTransportSuccess = false, } = opts; const label = logLabel ?? url; @@ -601,7 +604,9 @@ export async function instrumentedFetch( throw error; } const ms = Date.now() - start; - onTransportOutcome?.(); + if (!deferTransportSuccess) { + onTransportOutcome?.(); + } httpLog(method, label, response, ms); recordClientSpanStatus(span, response.status); diff --git a/packages/world-vercel/src/storage.ts b/packages/world-vercel/src/storage.ts index 186c759805..72abe6c57b 100644 --- a/packages/world-vercel/src/storage.ts +++ b/packages/world-vercel/src/storage.ts @@ -8,6 +8,7 @@ import { createWorkflowRunEventBatch, getEvent, getWorkflowRunEvents, + getWorkflowRunEventsByCorrelationId, } from './events.js'; import { getHook, getHookByToken, listHooks } from './hooks.js'; import { instrumentObject } from './instrumentObject.js'; @@ -53,7 +54,8 @@ export function createStorage(config?: APIConfig): Storage { createWorkflowRunEventBatch(runId, events, params, config), get: (runId, eventId, params) => getEvent(runId, eventId, params, config), list: (params) => getWorkflowRunEvents(params, config), - listByCorrelationId: (params) => getWorkflowRunEvents(params, config), + listByCorrelationId: (params) => + getWorkflowRunEventsByCorrelationId(params, config), }, hooks: { get: (hookId, params) => getHook(hookId, params, config), diff --git a/packages/world-vercel/src/trace-propagation.test.ts b/packages/world-vercel/src/trace-propagation.test.ts index b22b2d0ec6..a8155c6f5d 100644 --- a/packages/world-vercel/src/trace-propagation.test.ts +++ b/packages/world-vercel/src/trace-propagation.test.ts @@ -166,7 +166,7 @@ describe('v4 event requests (fetchV4) trace propagation', () => { agent .get(origin) .intercept({ - path: '/api/v4/runs/wrun_1/events?returnAll=true', + path: '/api/v4/runs/wrun_1/events?remoteRefBehavior=resolve&returnAll=true', method: 'GET', }) .reply(200, encodeFrame({ _end: 1, hasMore: false }, new Uint8Array(0)), { @@ -184,8 +184,7 @@ describe('v4 event requests (fetchV4) trace propagation', () => { traceId = span.spanContext().traceId; spanId = span.spanContext().spanId; await getWorkflowRunEventsV4( - 'wrun_1', - {}, + { runId: 'wrun_1' }, { token: 'test-token', dispatcher: agent } ); span.end(); diff --git a/packages/world/src/events.ts b/packages/world/src/events.ts index 344462ea22..defec5e0f4 100644 --- a/packages/world/src/events.ts +++ b/packages/world/src/events.ts @@ -1141,6 +1141,7 @@ export interface ListEventsByCorrelationIdParams { * event id alone is not. */ runId: string; + /** Omit `limit` to return every remaining event. */ pagination?: PaginationOptions; resolveData?: ResolveData; }