From 085022c8debb90fa9f0296e1e2bb9afa89541e00 Mon Sep 17 00:00:00 2001 From: Nathan Colosimo <110621881+NathanColosimo@users.noreply.github.com> Date: Thu, 13 Aug 2026 20:12:10 -0700 Subject: [PATCH 1/9] Resume partial replay event streams --- .changeset/resume-partial-replay-streams.md | 6 + packages/world-vercel/src/events-v4.test.ts | 215 ++++++++++++++++++++ packages/world-vercel/src/events-v4.ts | 144 +++++++++++-- packages/world-vercel/src/events.test.ts | 59 +++++- packages/world-vercel/src/events.ts | 12 +- packages/world/src/events.ts | 16 ++ 6 files changed, 417 insertions(+), 35 deletions(-) create mode 100644 .changeset/resume-partial-replay-streams.md diff --git a/.changeset/resume-partial-replay-streams.md b/.changeset/resume-partial-replay-streams.md new file mode 100644 index 0000000000..9f5d35e640 --- /dev/null +++ b/.changeset/resume-partial-replay-streams.md @@ -0,0 +1,6 @@ +--- +"@workflow/world": patch +"@workflow/world-vercel": patch +--- + +Resume interrupted or partial replay event streams after their last validated event and expose decoded events to streaming consumers. diff --git a/packages/world-vercel/src/events-v4.test.ts b/packages/world-vercel/src/events-v4.test.ts index 911e1a57e3..503e0a3475 100644 --- a/packages/world-vercel/src/events-v4.test.ts +++ b/packages/world-vercel/src/events-v4.test.ts @@ -891,6 +891,221 @@ describe('createWorkflowRunEventV4 over HTTP', () => { agent.assertNoPendingInterceptors(); }); + it('continues a truncated run_started stream after its last event', async () => { + const origin = + WORKFLOW_SERVER_URL_OVERRIDE || 'https://vercel-workflow.com'; + const agent = new MockAgent(); + agent.disableNetConnect(); + const observed: string[] = []; + + agent + .get(origin) + .intercept({ + path: '/api/v4/runs/wrun_1/events/run_started', + method: 'POST', + headers: { accept: V4_FRAME_CONTENT_TYPE }, + }) + .reply( + 200, + encodeFrame( + { + eventId: 'evnt_1', + runId: 'wrun_1', + eventType: 'run_created', + createdAt: CREATED_AT, + eventData: { + deploymentId: 'dpl_1', + workflowName: 'workflow', + input: null, + }, + }, + new Uint8Array() + ), + { + headers: { + 'content-type': V4_FRAME_CONTENT_TYPE, + 'x-wf-max-events': '10000', + }, + } + ); + agent + .get(origin) + .intercept({ + path: '/api/v4/runs/wrun_1/events?returnAll=true&cursor=eid%3Aevnt_1', + method: 'GET', + }) + .reply( + 200, + Buffer.concat([ + encodeFrame( + { + eventId: 'evnt_2', + runId: 'wrun_1', + eventType: 'run_started', + createdAt: CREATED_AT, + }, + new Uint8Array() + ), + encodeFrame( + { _end: 1, next: 'eid:evnt_2', hasMore: false }, + new Uint8Array() + ), + ]), + { headers: { 'content-type': V4_FRAME_CONTENT_TYPE } } + ); + + const result = await createWorkflowRunStartedEventV4( + { runId: 'wrun_1', specVersion: 5 }, + { token: 'test-token', dispatcher: agent }, + (event) => observed.push(event.eventId) + ); + + expect(result.events.map((event) => event.eventId)).toEqual([ + 'evnt_1', + 'evnt_2', + ]); + expect(result.hasMore).toBe(false); + expect(observed).toEqual(['evnt_1', 'evnt_2']); + agent.assertNoPendingInterceptors(); + }); + + it('does not treat an event observer failure as stream truncation', async () => { + const origin = + WORKFLOW_SERVER_URL_OVERRIDE || 'https://vercel-workflow.com'; + const agent = new MockAgent(); + agent.disableNetConnect(); + + agent + .get(origin) + .intercept({ + path: '/api/v4/runs/wrun_1/events/run_started', + method: 'POST', + headers: { accept: V4_FRAME_CONTENT_TYPE }, + }) + .reply( + 200, + Buffer.concat([ + encodeFrame( + { + eventId: 'evnt_1', + runId: 'wrun_1', + eventType: 'run_created', + createdAt: CREATED_AT, + eventData: { + deploymentId: 'dpl_1', + workflowName: 'workflow', + input: null, + }, + }, + new Uint8Array() + ), + encodeFrame( + { _end: 1, next: 'eid:evnt_1', hasMore: false }, + new Uint8Array() + ), + ]), + { + headers: { + 'content-type': V4_FRAME_CONTENT_TYPE, + 'x-wf-max-events': '10000', + }, + } + ); + + await expect( + createWorkflowRunStartedEventV4( + { runId: 'wrun_1', specVersion: 5 }, + { token: 'test-token', dispatcher: agent }, + () => { + throw new Error('observer failed'); + } + ) + ).rejects.toThrow('observer failed'); + agent.assertNoPendingInterceptors(); + }); + + it('continues a graceful partial run_started stream from its sentinel cursor', async () => { + const origin = + WORKFLOW_SERVER_URL_OVERRIDE || 'https://vercel-workflow.com'; + const agent = new MockAgent(); + agent.disableNetConnect(); + + agent + .get(origin) + .intercept({ + path: '/api/v4/runs/wrun_1/events/run_started', + method: 'POST', + headers: { accept: V4_FRAME_CONTENT_TYPE }, + }) + .reply( + 200, + Buffer.concat([ + encodeFrame( + { + eventId: 'evnt_1', + runId: 'wrun_1', + eventType: 'run_created', + createdAt: CREATED_AT, + eventData: { + deploymentId: 'dpl_1', + workflowName: 'workflow', + input: null, + }, + }, + new Uint8Array() + ), + encodeFrame( + { _end: 1, next: 'eid:evnt_1', hasMore: true }, + new Uint8Array() + ), + ]), + { + headers: { + 'content-type': V4_FRAME_CONTENT_TYPE, + 'x-wf-max-events': '10000', + }, + } + ); + agent + .get(origin) + .intercept({ + path: '/api/v4/runs/wrun_1/events?returnAll=true&cursor=eid%3Aevnt_1', + method: 'GET', + }) + .reply( + 200, + Buffer.concat([ + encodeFrame( + { + eventId: 'evnt_2', + runId: 'wrun_1', + eventType: 'run_started', + createdAt: CREATED_AT, + }, + new Uint8Array() + ), + encodeFrame( + { _end: 1, next: 'eid:evnt_2', hasMore: false }, + new Uint8Array() + ), + ]), + { headers: { 'content-type': V4_FRAME_CONTENT_TYPE } } + ); + + const result = await createWorkflowRunStartedEventV4( + { runId: 'wrun_1', specVersion: 5 }, + { token: 'test-token', dispatcher: agent } + ); + + expect(result.events.map((event) => event.eventId)).toEqual([ + 'evnt_1', + 'evnt_2', + ]); + expect(result.cursor).toBe('eid:evnt_2'); + expect(result.hasMore).toBe(false); + agent.assertNoPendingInterceptors(); + }); + it('requires the event-stream response requested by run_started', async () => { const origin = WORKFLOW_SERVER_URL_OVERRIDE || 'https://vercel-workflow.com'; diff --git a/packages/world-vercel/src/events-v4.ts b/packages/world-vercel/src/events-v4.ts index 2b9f8f0fe2..c626218109 100644 --- a/packages/world-vercel/src/events-v4.ts +++ b/packages/world-vercel/src/events-v4.ts @@ -27,6 +27,7 @@ import { type Event, type EventResult, EventSchema, + type EventStreamObserver, type EventType, EventTypeSchema, getEventDataPayloadField, @@ -779,7 +780,8 @@ async function decodeCreateEventResponse( export async function createWorkflowRunStartedEventV4( input: CreateEventV4InputBase, - config?: APIConfig + config?: APIConfig, + onEvent?: EventStreamObserver ) { const response = await postWorkflowRunEventV4( { ...input, eventType: 'run_started' }, @@ -787,7 +789,14 @@ export async function createWorkflowRunStartedEventV4( config ); const events: Event[] = []; - const page = await consumeEventFrameStream(response, 'createEvent', events); + const page = await consumeReplayLogResponse({ + response, + runId: input.runId, + opName: 'createEvent', + events, + config, + onEvent, + }); assert(page.cursor, 'v4 createEvent: event stream missing cursor'); const maxEvents = MaxEventsHeaderSchema.safeParse( response.headers.get(MAX_EVENTS_HEADER) @@ -1218,13 +1227,14 @@ export type HookReceivedPreloadV4Result = * A server that supports the lazy-hook replay stream answers the consumer's * idempotent re-ensure with the run's complete replay log as v4 frames — * the same event-frame sequence LIST uses, ending with the `_end` sentinel. - * A truncated stream (EOF without the sentinel) throws; the write is - * deduplicated by the server's `(runId, resumeId)` constraint, so retrying - * the whole request is safe and converges on the same canonical event. + * A truncated stream resumes with a GET after its last validated event. If it + * fails before producing any event, the outer POST retry remains safe because + * the server deduplicates `(runId, resumeId)` and returns the canonical event. */ export async function createHookReceivedPreloadEventV4( input: CreateEventV4InputBase, - config?: APIConfig + config?: APIConfig, + onEvent?: EventStreamObserver ): Promise { const response = await postWorkflowRunEventV4( { ...input, eventType: 'hook_received' }, @@ -1241,7 +1251,14 @@ export async function createHookReceivedPreloadEventV4( } const events: Event[] = []; - const page = await consumeEventFrameStream(response, 'createEvent', events); + const page = await consumeReplayLogResponse({ + response, + runId: input.runId, + opName: 'createEvent', + events, + config, + onEvent, + }); const maxEvents = MaxEventsHeaderSchema.safeParse( response.headers.get(MAX_EVENTS_HEADER) ); @@ -1319,6 +1336,8 @@ export interface ListEventsV4Params extends PaginationOptions { * it. */ remoteRefBehavior?: 'resolve' | 'lazy'; + /** Called synchronously after each event frame validates. */ + onEvent?: EventStreamObserver; } export interface ListEventsV4Result { @@ -1332,7 +1351,8 @@ export interface ListEventsV4Result { async function consumeEventFrameStream( response: Response, opName: string, - events: Event[] + events: Event[], + onEvent?: EventStreamObserver ): Promise> { const contentType = response.headers.get('content-type'); if (!contentType?.startsWith(V4_FRAME_CONTENT_TYPE)) { @@ -1342,24 +1362,91 @@ async function consumeEventFrameStream( } const chunks = response.body as unknown as AsyncIterable; + const frames = decodeFrames(chunks)[Symbol.asyncIterator](); - for await (const frame of decodeFrames(chunks)) { - if (frame.meta._end === 1) { - const end = EventStreamEndSchema.parse(frame.meta); - return { cursor: end.next ?? null, hasMore: end.hasMore }; - } - if (Object.keys(frame.meta).some((key) => key.startsWith('_'))) { - throw new Error(`v4 ${opName}: unexpected control frame`); + try { + for (;;) { + let next: IteratorResult; + try { + next = await frames.next(); + } catch (cause) { + throw new WorkflowWorldError( + `v4 ${opName}: event frame stream failed after ${events.length} events`, + { code: 'TRANSPORT', cause } + ); + } + if (next.done) break; + + const frame = next.value; + if (frame.meta._end === 1) { + const end = EventStreamEndSchema.parse(frame.meta); + return { cursor: end.next ?? null, hasMore: end.hasMore }; + } + if (Object.keys(frame.meta).some((key) => key.startsWith('_'))) { + throw new Error(`v4 ${opName}: unexpected control frame`); + } + const event = decodeEventFrame(frame); + events.push(event); + // Deliberately outside the stream-read catch above: observer/application + // failures are not truncation and must never advance recovery past this + // event. + onEvent?.(event); } - events.push(decodeEventFrame(frame)); + } finally { + await frames.return?.(undefined); } - throw new Error( + throw new WorkflowWorldError( `v4 ${opName}: frame stream ended without the end-of-stream sentinel ` + - `(${events.length} events read) — truncated response?` + `(${events.length} events read) — truncated response?`, + { code: 'TRANSPORT' } ); } +/** + * Finish a replay-log response, reusing every validated prefix. A graceful + * `hasMore` sentinel and an interrupted body now converge on the same GET + * continuation instead of making run_started download its accepted prefix + * again. + */ +async function consumeReplayLogResponse({ + response, + runId, + opName, + events, + config, + onEvent, +}: { + response: Response; + runId: string; + opName: string; + events: Event[]; + config?: APIConfig; + onEvent?: EventStreamObserver; +}): Promise> { + let page: Pick; + try { + page = await consumeEventFrameStream(response, opName, events, onEvent); + } catch (error) { + if (!WorkflowWorldError.is(error) || error.code !== 'TRANSPORT') { + throw error; + } + const lastEvent = events.at(-1); + if (!lastEvent) throw error; + page = { cursor: `eid:${lastEvent.eventId}`, hasMore: true }; + } + + if (!page.hasMore) return page; + assert(page.cursor, `v4 ${opName}: partial event stream missing cursor`); + const suffix = await getWorkflowRunEventsV4( + runId, + { cursor: page.cursor, onEvent }, + config + ); + events.push(...suffix.events); + return { cursor: suffix.cursor, hasMore: suffix.hasMore }; +} + /** * Drive a v4 frame-stream list response into an in-memory page. Used by * both the by-runId and by-correlationId list endpoints — the wire @@ -1374,7 +1461,8 @@ async function consumeListFrameStream( headers: Headers, config: APIConfig | undefined, opName: string, - events: Event[] + events: Event[], + onEvent?: EventStreamObserver ): Promise> { const response = await fetchV4( url, @@ -1382,7 +1470,7 @@ async function consumeListFrameStream( config, opName ); - return consumeEventFrameStream(response, opName, events); + return consumeEventFrameStream(response, opName, events, onEvent); } /** @@ -1436,8 +1524,19 @@ export async function getWorkflowRunEventsV4( headers, config, 'listEvents', - events + events, + params.onEvent ); + if (params.limit === undefined && page.hasMore) { + if (!page.cursor || page.cursor === cursor) { + throw new WorkflowWorldError( + `v4 listEvents: partial event stream made no cursor progress for run ${runId}`, + { code: 'SCHEMA_VALIDATION' } + ); + } + cursor = page.cursor; + continue; + } return { events, ...page }; } catch (error) { const lastEvent = events.at(-1); @@ -1485,7 +1584,8 @@ export async function getEventsByCorrelationIdV4( headers, config, 'listEventsByCorrelationId', - events + events, + params.onEvent ); return { events, ...page }; } diff --git a/packages/world-vercel/src/events.test.ts b/packages/world-vercel/src/events.test.ts index 5fea9675a4..d072603000 100644 --- a/packages/world-vercel/src/events.test.ts +++ b/packages/world-vercel/src/events.test.ts @@ -1393,7 +1393,14 @@ describe('getWorkflowRunEvents legacy structured-error compatibility', () => { * fallback preserves their (correct, if slower) behavior. */ describe('getWorkflowRunEvents hasMore mapping', () => { - function mockListResponse(agent: MockAgent, sentinelMeta: object) { + function mockListResponse( + agent: MockAgent, + sentinelMeta: object, + query: Record = { + returnAll: 'true', + remoteRefBehavior: 'resolve', + } + ) { const frames = Buffer.concat([ encodeFrame( { @@ -1412,9 +1419,7 @@ describe('getWorkflowRunEvents hasMore mapping', () => { .intercept({ path: '/api/v4/runs/wrun_1/events', method: 'GET', - // These tests omit the limit and use the default resolveData - // ('all' → resolve); match both translated query params. - query: { returnAll: 'true', remoteRefBehavior: 'resolve' }, + query, }) .reply(200, frames, { headers: { 'content-type': V4_FRAME_CONTENT_TYPE }, @@ -1439,10 +1444,14 @@ describe('getWorkflowRunEvents hasMore mapping', () => { it('maps an explicit hasMore:true through', async () => { const agent = mockAgent(); - mockListResponse(agent, { _end: 1, next: 'cursor-2', hasMore: true }); + mockListResponse( + agent, + { _end: 1, next: 'cursor-2', hasMore: true }, + { limit: '500', remoteRefBehavior: 'resolve' } + ); const result = await getWorkflowRunEvents( - { runId: 'wrun_1' }, + { runId: 'wrun_1', pagination: { limit: 500 } }, { token: 'test-token', dispatcher: agent } ); @@ -1842,7 +1851,7 @@ describe('createWorkflowRunEvent hook_received replay preload', () => { agent.assertNoPendingInterceptors(); }); - it('rejects a truncated preload stream (no end sentinel)', async () => { + it('continues a truncated preload stream after its last event', async () => { const agent = mockAgent(); agent .get(ORIGIN) @@ -1866,15 +1875,43 @@ describe('createWorkflowRunEvent hook_received replay preload', () => { }, PAYLOAD ), + { + headers: { + 'content-type': V4_FRAME_CONTENT_TYPE, + 'x-wf-event-id': 'evnt_4', + }, + } + ); + + agent + .get(ORIGIN) + .intercept({ + path: '/api/v4/runs/wrun_1/events?returnAll=true&cursor=eid%3Aevnt_4', + method: 'GET', + }) + .reply( + 200, + encodeFrame( + { _end: 1, next: 'eid:evnt_4', hasMore: false }, + new Uint8Array() + ), { headers: { 'content-type': V4_FRAME_CONTENT_TYPE } } ); - await expect( - createWorkflowRunEvent('wrun_1', hookReceivedRequest(), preloadParams, { + const result = await createWorkflowRunEvent( + 'wrun_1', + hookReceivedRequest(), + preloadParams, + { token: 'test-token', dispatcher: agent, - }) - ).rejects.toThrow(/end-of-stream sentinel/); + } + ); + + expect(result.event?.eventId).toBe('evnt_4'); + expect(result.events).toHaveLength(1); + expect(result.cursor).toBe('eid:evnt_4'); + expect(result.hasMore).toBe(false); agent.assertNoPendingInterceptors(); }); diff --git a/packages/world-vercel/src/events.ts b/packages/world-vercel/src/events.ts index 9706a59af9..480115e2dd 100644 --- a/packages/world-vercel/src/events.ts +++ b/packages/world-vercel/src/events.ts @@ -446,6 +446,9 @@ export async function getWorkflowRunEvents( const listParams: ListEventsV4Params = { ...pagination, remoteRefBehavior: resolveData === 'none' ? 'lazy' : 'resolve', + ...('onEvent' in params && params.onEvent + ? { onEvent: params.onEvent } + : {}), }; const result = await ('correlationId' in params @@ -752,7 +755,11 @@ async function createWorkflowRunEventInner( }; if (data.eventType === 'run_started' && !params?.skipPreload) { - const result = await createWorkflowRunStartedEventV4(input, config); + const result = await createWorkflowRunStartedEventV4( + input, + config, + params?.onEvent + ); const runCreated = result.events.find( (event) => event.eventType === 'run_created' ); @@ -819,7 +826,8 @@ async function createWorkflowRunEventInner( // an S3-backed hook payload the runtime would discard anyway. const outcome = await createHookReceivedPreloadEventV4( { ...input, remoteRefBehavior: 'lazy' }, - config + config, + params.onEvent ); if (outcome.kind === 'materialized') { // Older server (or optimization declined): the write still succeeded diff --git a/packages/world/src/events.ts b/packages/world/src/events.ts index 344462ea22..fc222c64cc 100644 --- a/packages/world/src/events.ts +++ b/packages/world/src/events.ts @@ -724,6 +724,15 @@ export type HookCreatedEventRequest = EventRequestOfType<'hook_created'>; export type HookReceivedEvent = z.infer; export type HookConflictEvent = z.infer; +/** + * Local observer invoked as each event is decoded from a streamed response. + * Worlds without a streaming implementation may ignore it. The callback runs + * synchronously, so its work intentionally applies response-stream backpressure. + * Keep it bounded to the event just decoded; large payloads may proportionally + * delay the next frame. + */ +export type EventStreamObserver = (event: Event) => void; + /** * Union of all possible event request types. * @internal Use CreateEventRequest or RunCreatedEventRequest instead. @@ -941,6 +950,11 @@ export interface CreateEventParams { * `resumeHook()` must not set it. */ preloadEvents?: true; + /** + * Observe replay-preload events as their frames are decoded. This is a + * client-side delivery hook only; it is never serialized to a backend. + */ + onEvent?: EventStreamObserver; } /** @@ -1128,6 +1142,8 @@ export interface ListEventsParams { /** Omit `limit` to return every remaining event. */ pagination?: PaginationOptions; resolveData?: ResolveData; + /** Observe events as a streaming World decodes them. */ + onEvent?: EventStreamObserver; } export interface ListEventsByCorrelationIdParams { From 48d133f9f61d3464f117fda95d9106185c273c6a Mon Sep 17 00:00:00 2001 From: Nathan Colosimo <110621881+NathanColosimo@users.noreply.github.com> Date: Thu, 13 Aug 2026 23:53:30 -0700 Subject: [PATCH 2/9] [world-vercel] Preserve event observer failures --- packages/world-vercel/src/events-v4.test.ts | 46 +++++++++++++++++++++ packages/world-vercel/src/events-v4.ts | 9 ++-- 2 files changed, 52 insertions(+), 3 deletions(-) diff --git a/packages/world-vercel/src/events-v4.test.ts b/packages/world-vercel/src/events-v4.test.ts index 503e0a3475..eb3342f145 100644 --- a/packages/world-vercel/src/events-v4.test.ts +++ b/packages/world-vercel/src/events-v4.test.ts @@ -316,6 +316,52 @@ describe('getWorkflowRunEventsV4 over HTTP', () => { agent.assertNoPendingInterceptors(); }); + it('does not resume past a GET observer failure', async () => { + const origin = + WORKFLOW_SERVER_URL_OVERRIDE || 'https://vercel-workflow.com'; + const agent = new MockAgent(); + agent.disableNetConnect(); + + agent + .get(origin) + .intercept({ + path: '/api/v4/runs/wrun_1/events?returnAll=true', + method: 'GET', + }) + .reply( + 200, + Buffer.concat([ + encodeFrame( + { + eventId: 'evnt_1', + runId: 'wrun_1', + eventType: 'run_started', + createdAt: CREATED_AT, + }, + new Uint8Array() + ), + encodeFrame( + { _end: 1, next: 'eid:evnt_1', hasMore: false }, + new Uint8Array() + ), + ]), + { headers: { 'content-type': V4_FRAME_CONTENT_TYPE } } + ); + + await expect( + getWorkflowRunEventsV4( + 'wrun_1', + { + onEvent: () => { + throw new Error('observer failed'); + }, + }, + { token: 'test-token', dispatcher: agent } + ) + ).rejects.toThrow('observer failed'); + agent.assertNoPendingInterceptors(); + }); + it.each([ ['an unknown event type', { eventType: 'future_event', eventData: {} }], ['invalid event metadata', { eventType: 'run_created', eventData: {} }], diff --git a/packages/world-vercel/src/events-v4.ts b/packages/world-vercel/src/events-v4.ts index c626218109..875d8a575a 100644 --- a/packages/world-vercel/src/events-v4.ts +++ b/packages/world-vercel/src/events-v4.ts @@ -1348,6 +1348,10 @@ export interface ListEventsV4Result { hasMore: boolean; } +function isTransportError(error: unknown): error is WorkflowWorldError { + return WorkflowWorldError.is(error) && error.code === 'TRANSPORT'; +} + async function consumeEventFrameStream( response: Response, opName: string, @@ -1428,9 +1432,7 @@ async function consumeReplayLogResponse({ try { page = await consumeEventFrameStream(response, opName, events, onEvent); } catch (error) { - if (!WorkflowWorldError.is(error) || error.code !== 'TRANSPORT') { - throw error; - } + if (!isTransportError(error)) throw error; const lastEvent = events.at(-1); if (!lastEvent) throw error; page = { cursor: `eid:${lastEvent.eventId}`, hasMore: true }; @@ -1539,6 +1541,7 @@ export async function getWorkflowRunEventsV4( } return { events, ...page }; } catch (error) { + if (!isTransportError(error)) throw error; const lastEvent = events.at(-1); if ( params.limit !== undefined || From a6aa75f3ae90a1bd23cd249459fb3027ff14099a Mon Sep 17 00:00:00 2001 From: Nathan Colosimo <110621881+NathanColosimo@users.noreply.github.com> Date: Fri, 14 Aug 2026 12:13:51 -0700 Subject: [PATCH 3/9] [world-vercel] Simplify replay stream handling --- packages/world-vercel/src/events-v4.test.ts | 14 ++-- packages/world-vercel/src/events-v4.ts | 72 ++++++++++----------- packages/world-vercel/src/events.test.ts | 3 +- packages/world-vercel/src/events.ts | 54 ++++++++++------ packages/world-vercel/src/storage.ts | 4 +- packages/world/src/events.ts | 11 ++-- 6 files changed, 90 insertions(+), 68 deletions(-) diff --git a/packages/world-vercel/src/events-v4.test.ts b/packages/world-vercel/src/events-v4.test.ts index eb3342f145..d63fc95d7e 100644 --- a/packages/world-vercel/src/events-v4.test.ts +++ b/packages/world-vercel/src/events-v4.test.ts @@ -348,17 +348,20 @@ describe('getWorkflowRunEventsV4 over HTTP', () => { { headers: { 'content-type': V4_FRAME_CONTENT_TYPE } } ); + const observerError = new WorkflowWorldError('observer failed', { + code: 'TRANSPORT', + }); await expect( getWorkflowRunEventsV4( 'wrun_1', { onEvent: () => { - throw new Error('observer failed'); + throw observerError; }, }, { token: 'test-token', dispatcher: agent } ) - ).rejects.toThrow('observer failed'); + ).rejects.toBe(observerError); agent.assertNoPendingInterceptors(); }); @@ -1058,15 +1061,18 @@ describe('createWorkflowRunEventV4 over HTTP', () => { } ); + const observerError = new WorkflowWorldError('observer failed', { + code: 'TRANSPORT', + }); await expect( createWorkflowRunStartedEventV4( { runId: 'wrun_1', specVersion: 5 }, { token: 'test-token', dispatcher: agent }, () => { - throw new Error('observer failed'); + throw observerError; } ) - ).rejects.toThrow('observer failed'); + ).rejects.toBe(observerError); agent.assertNoPendingInterceptors(); }); diff --git a/packages/world-vercel/src/events-v4.ts b/packages/world-vercel/src/events-v4.ts index 875d8a575a..44c62f23b8 100644 --- a/packages/world-vercel/src/events-v4.ts +++ b/packages/world-vercel/src/events-v4.ts @@ -1348,8 +1348,25 @@ export interface ListEventsV4Result { hasMore: boolean; } -function isTransportError(error: unknown): error is WorkflowWorldError { - return WorkflowWorldError.is(error) && error.code === 'TRANSPORT'; +class PartialEventStreamError extends WorkflowWorldError { + constructor(message: string, cause?: unknown) { + super(message, { code: 'TRANSPORT', cause }); + } +} + +async function* readEventFrames( + chunks: AsyncIterable, + opName: string, + events: readonly Event[] +): AsyncGenerator { + try { + yield* decodeFrames(chunks); + } catch (cause) { + throw new PartialEventStreamError( + `v4 ${opName}: event frame stream failed after ${events.length} events`, + cause + ); + } } async function consumeEventFrameStream( @@ -1366,44 +1383,25 @@ async function consumeEventFrameStream( } const chunks = response.body as unknown as AsyncIterable; - const frames = decodeFrames(chunks)[Symbol.asyncIterator](); - - try { - for (;;) { - let next: IteratorResult; - try { - next = await frames.next(); - } catch (cause) { - throw new WorkflowWorldError( - `v4 ${opName}: event frame stream failed after ${events.length} events`, - { code: 'TRANSPORT', cause } - ); - } - if (next.done) break; - const frame = next.value; - if (frame.meta._end === 1) { - const end = EventStreamEndSchema.parse(frame.meta); - return { cursor: end.next ?? null, hasMore: end.hasMore }; - } - if (Object.keys(frame.meta).some((key) => key.startsWith('_'))) { - throw new Error(`v4 ${opName}: unexpected control frame`); - } - const event = decodeEventFrame(frame); - events.push(event); - // Deliberately outside the stream-read catch above: observer/application - // failures are not truncation and must never advance recovery past this - // event. - onEvent?.(event); + for await (const frame of readEventFrames(chunks, opName, events)) { + if (frame.meta._end === 1) { + const end = EventStreamEndSchema.parse(frame.meta); + return { cursor: end.next ?? null, hasMore: end.hasMore }; + } + if (Object.keys(frame.meta).some((key) => key.startsWith('_'))) { + throw new Error(`v4 ${opName}: unexpected control frame`); } - } finally { - await frames.return?.(undefined); + const event = decodeEventFrame(frame); + events.push(event); + // Observer/application failures remain outside the frame reader, so they + // are not mistaken for truncation or skipped by partial-stream recovery. + onEvent?.(event); } - throw new WorkflowWorldError( + throw new PartialEventStreamError( `v4 ${opName}: frame stream ended without the end-of-stream sentinel ` + - `(${events.length} events read) — truncated response?`, - { code: 'TRANSPORT' } + `(${events.length} events read) — truncated response?` ); } @@ -1432,7 +1430,7 @@ async function consumeReplayLogResponse({ try { page = await consumeEventFrameStream(response, opName, events, onEvent); } catch (error) { - if (!isTransportError(error)) throw error; + if (!(error instanceof PartialEventStreamError)) throw error; const lastEvent = events.at(-1); if (!lastEvent) throw error; page = { cursor: `eid:${lastEvent.eventId}`, hasMore: true }; @@ -1541,7 +1539,7 @@ export async function getWorkflowRunEventsV4( } return { events, ...page }; } catch (error) { - if (!isTransportError(error)) throw error; + if (!(error instanceof PartialEventStreamError)) throw error; const lastEvent = events.at(-1); if ( params.limit !== undefined || diff --git a/packages/world-vercel/src/events.test.ts b/packages/world-vercel/src/events.test.ts index d072603000..95bad148ab 100644 --- a/packages/world-vercel/src/events.test.ts +++ b/packages/world-vercel/src/events.test.ts @@ -8,6 +8,7 @@ import { describe, expect, it } from 'vitest'; import { createWorkflowRunEvent, getWorkflowRunEvents, + getWorkflowRunEventsByCorrelationId, splitEventDataForV4, } from './events.js'; import { encodeFrame, V4_FRAME_CONTENT_TYPE } from './frames.js'; @@ -1520,7 +1521,7 @@ describe('getWorkflowRunEvents by correlation id is scoped to the run', () => { headers: { 'content-type': V4_FRAME_CONTENT_TYPE }, }); - const result = await getWorkflowRunEvents( + const result = await getWorkflowRunEventsByCorrelationId( { correlationId: 'step_001', runId: 'wrun_1' }, { token: 'test-token', dispatcher: agent } ); diff --git a/packages/world-vercel/src/events.ts b/packages/world-vercel/src/events.ts index 480115e2dd..4c114cf45f 100644 --- a/packages/world-vercel/src/events.ts +++ b/packages/world-vercel/src/events.ts @@ -48,6 +48,7 @@ import { getEventDataPayloadField, isHookEventRequiringExistence, type ListEventsByCorrelationIdParams, + type ListEventsOptions, type ListEventsParams, type PaginatedResponse, validateUlidTimestamp, @@ -435,30 +436,46 @@ export async function getEvent( ); } -export async function getWorkflowRunEvents( - params: ListEventsParams | ListEventsByCorrelationIdParams, - config?: APIConfig -): Promise> { +function getListEventsV4Params(params: ListEventsOptions): ListEventsV4Params { const { pagination, resolveData = DEFAULT_RESOLVE_DATA_OPTION } = params; // `resolveData: 'none'` leaves payload refs unresolved, so the backend can // skip reading and streaming their contents. The validated lazy descriptors // remain on the returned events. - const listParams: ListEventsV4Params = { + return { ...pagination, remoteRefBehavior: resolveData === 'none' ? 'lazy' : 'resolve', - ...('onEvent' in params && params.onEvent - ? { onEvent: params.onEvent } - : {}), }; +} + +export async function getWorkflowRunEvents( + params: ListEventsParams, + config?: APIConfig +): Promise> { + const result = await getWorkflowRunEventsV4( + params.runId, + { ...getListEventsV4Params(params), onEvent: params.onEvent }, + config + ); + + return { + data: result.events, + // The cursor is present even on the final page because it is also the + // incremental-load resume point. `hasMore` is the pagination signal. + cursor: result.cursor, + hasMore: result.hasMore, + }; +} - const result = await ('correlationId' in params - ? getEventsByCorrelationIdV4( - params.correlationId, - params.runId, - listParams, - config - ) - : getWorkflowRunEventsV4(params.runId, listParams, config)); +export async function getWorkflowRunEventsByCorrelationId( + params: ListEventsByCorrelationIdParams, + config?: APIConfig +): Promise> { + const result = await getEventsByCorrelationIdV4( + params.correlationId, + params.runId, + getListEventsV4Params(params), + config + ); // A correlation id is unique per run, not globally — a slot-numbered run // numbers its own steps, so `step_…001` names the first step of every such @@ -467,10 +484,7 @@ export async function getWorkflowRunEvents( // `hasMore`/`cursor` stay the backend's, so a page that filters down to // nothing is still followed by the next one. return { - data: - 'correlationId' in params - ? result.events.filter((event) => event.runId === params.runId) - : result.events, + data: result.events.filter((event) => event.runId === params.runId), // The cursor is present even on the final page because it is also the // incremental-load resume point. `hasMore` is the pagination signal. cursor: result.cursor, diff --git a/packages/world-vercel/src/storage.ts b/packages/world-vercel/src/storage.ts index 186c759805..72abe6c57b 100644 --- a/packages/world-vercel/src/storage.ts +++ b/packages/world-vercel/src/storage.ts @@ -8,6 +8,7 @@ import { createWorkflowRunEventBatch, getEvent, getWorkflowRunEvents, + getWorkflowRunEventsByCorrelationId, } from './events.js'; import { getHook, getHookByToken, listHooks } from './hooks.js'; import { instrumentObject } from './instrumentObject.js'; @@ -53,7 +54,8 @@ export function createStorage(config?: APIConfig): Storage { createWorkflowRunEventBatch(runId, events, params, config), get: (runId, eventId, params) => getEvent(runId, eventId, params, config), list: (params) => getWorkflowRunEvents(params, config), - listByCorrelationId: (params) => getWorkflowRunEvents(params, config), + listByCorrelationId: (params) => + getWorkflowRunEventsByCorrelationId(params, config), }, hooks: { get: (hookId, params) => getHook(hookId, params, config), diff --git a/packages/world/src/events.ts b/packages/world/src/events.ts index fc222c64cc..5a1577e191 100644 --- a/packages/world/src/events.ts +++ b/packages/world/src/events.ts @@ -1137,16 +1137,19 @@ export interface GetEventParams { resolveData?: ResolveData; } -export interface ListEventsParams { - runId: string; +export interface ListEventsOptions { /** Omit `limit` to return every remaining event. */ pagination?: PaginationOptions; resolveData?: ResolveData; +} + +export interface ListEventsParams extends ListEventsOptions { + runId: string; /** Observe events as a streaming World decodes them. */ onEvent?: EventStreamObserver; } -export interface ListEventsByCorrelationIdParams { +export interface ListEventsByCorrelationIdParams extends ListEventsOptions { correlationId: string; /** * The run the correlation id belongs to. A correlation id is unique per @@ -1157,6 +1160,4 @@ export interface ListEventsByCorrelationIdParams { * event id alone is not. */ runId: string; - pagination?: PaginationOptions; - resolveData?: ResolveData; } From 2459710719ac7a58cff474576af7a19974391fab Mon Sep 17 00:00:00 2001 From: Nathan Colosimo <110621881+NathanColosimo@users.noreply.github.com> Date: Fri, 14 Aug 2026 15:36:32 -0700 Subject: [PATCH 4/9] [world-vercel] Reuse world event list types --- packages/world-vercel/src/events-v4.test.ts | 65 +++++----- packages/world-vercel/src/events-v4.ts | 120 ++++++++---------- packages/world-vercel/src/events.test.ts | 2 +- packages/world-vercel/src/events.ts | 42 +----- .../src/trace-propagation.test.ts | 5 +- packages/world/src/events.ts | 30 ++--- 6 files changed, 103 insertions(+), 161 deletions(-) diff --git a/packages/world-vercel/src/events-v4.test.ts b/packages/world-vercel/src/events-v4.test.ts index d63fc95d7e..a40526ca4f 100644 --- a/packages/world-vercel/src/events-v4.test.ts +++ b/packages/world-vercel/src/events-v4.test.ts @@ -294,7 +294,7 @@ describe('getWorkflowRunEventsV4 over HTTP', () => { agent .get(origin) .intercept({ - path: '/api/v4/runs/wrun_1/events?returnAll=true', + path: '/api/v4/runs/wrun_1/events?remoteRefBehavior=resolve&returnAll=true', method: 'GET', }) .reply(200, frames, { @@ -302,13 +302,12 @@ describe('getWorkflowRunEventsV4 over HTTP', () => { }); const result = await getWorkflowRunEventsV4( - 'wrun_1', - {}, + { runId: 'wrun_1' }, { token: 'test-token', dispatcher: agent } ); - expect(result.events).toHaveLength(1); - expect(result.events[0]).toMatchObject({ + expect(result.data).toHaveLength(1); + expect(result.data[0]).toMatchObject({ eventId: 'evnt_1', eventData: { input: body }, }); @@ -325,7 +324,7 @@ describe('getWorkflowRunEventsV4 over HTTP', () => { agent .get(origin) .intercept({ - path: '/api/v4/runs/wrun_1/events?returnAll=true', + path: '/api/v4/runs/wrun_1/events?remoteRefBehavior=resolve&returnAll=true', method: 'GET', }) .reply( @@ -353,8 +352,8 @@ describe('getWorkflowRunEventsV4 over HTTP', () => { }); await expect( getWorkflowRunEventsV4( - 'wrun_1', { + runId: 'wrun_1', onEvent: () => { throw observerError; }, @@ -377,7 +376,7 @@ describe('getWorkflowRunEventsV4 over HTTP', () => { agent .get(origin) .intercept({ - path: '/api/v4/runs/wrun_1/events?returnAll=true', + path: '/api/v4/runs/wrun_1/events?remoteRefBehavior=resolve&returnAll=true', method: 'GET', }) .reply( @@ -391,8 +390,7 @@ describe('getWorkflowRunEventsV4 over HTTP', () => { await expect( getWorkflowRunEventsV4( - 'wrun_1', - {}, + { runId: 'wrun_1' }, { token: 'test-token', dispatcher: agent } ) ).rejects.toThrow(); @@ -431,7 +429,7 @@ describe('getWorkflowRunEventsV4 over HTTP', () => { agent .get(origin) .intercept({ - path: '/api/v4/runs/wrun_1/events?returnAll=true', + path: '/api/v4/runs/wrun_1/events?remoteRefBehavior=resolve&returnAll=true', method: 'GET', }) .reply(200, frames, { @@ -439,8 +437,7 @@ describe('getWorkflowRunEventsV4 over HTTP', () => { }); const result = await getWorkflowRunEventsV4( - 'wrun_1', - {}, + { runId: 'wrun_1' }, { token: 'test-token', dispatcher: agent } ); @@ -462,7 +459,7 @@ describe('getWorkflowRunEventsV4 over HTTP', () => { agent .get(origin) .intercept({ - path: '/api/v4/runs/wrun_1/events?returnAll=true', + path: '/api/v4/runs/wrun_1/events?remoteRefBehavior=resolve&returnAll=true', method: 'GET', }) .reply(200, frames, { @@ -471,8 +468,7 @@ describe('getWorkflowRunEventsV4 over HTTP', () => { await expect( getWorkflowRunEventsV4( - 'wrun_1', - {}, + { runId: 'wrun_1' }, { token: 'test-token', dispatcher: agent } ) ).rejects.toThrow(); @@ -505,7 +501,7 @@ describe('getWorkflowRunEventsV4 over HTTP', () => { agent .get(origin) .intercept({ - path: '/api/v4/runs/wrun_1/events?limit=500', + path: '/api/v4/runs/wrun_1/events?limit=500&remoteRefBehavior=resolve', method: 'GET', }) .reply(200, frames, { @@ -514,8 +510,7 @@ describe('getWorkflowRunEventsV4 over HTTP', () => { await expect( getWorkflowRunEventsV4( - 'wrun_1', - { limit: 500 }, + { runId: 'wrun_1', pagination: { limit: 500 } }, { token: 'test-token', dispatcher: agent } ) ).rejects.toThrow(/end-of-stream sentinel/); @@ -530,7 +525,7 @@ describe('getWorkflowRunEventsV4 over HTTP', () => { agent .get(origin) .intercept({ - path: '/api/v4/runs/wrun_1/events?returnAll=true', + path: '/api/v4/runs/wrun_1/events?remoteRefBehavior=resolve&returnAll=true', method: 'GET', }) .reply( @@ -554,7 +549,7 @@ describe('getWorkflowRunEventsV4 over HTTP', () => { agent .get(origin) .intercept({ - path: '/api/v4/runs/wrun_1/events?returnAll=true&cursor=eid%3Aevnt_1', + path: '/api/v4/runs/wrun_1/events?cursor=eid%3Aevnt_1&remoteRefBehavior=resolve&returnAll=true', method: 'GET', }) .reply( @@ -578,12 +573,11 @@ describe('getWorkflowRunEventsV4 over HTTP', () => { ); const result = await getWorkflowRunEventsV4( - 'wrun_1', - {}, + { runId: 'wrun_1' }, { token: 'test-token', dispatcher: agent } ); - expect(result.events.map((event) => event.eventId)).toEqual([ + expect(result.data.map((event) => event.eventId)).toEqual([ 'evnt_1', 'evnt_2', ]); @@ -638,9 +632,11 @@ describe('getEventsByCorrelationIdV4 over HTTP', () => { }); const result = await getEventsByCorrelationIdV4( - 'step_001', - 'wrun_1', - { limit: 10 }, + { + correlationId: 'step_001', + runId: 'wrun_1', + pagination: { limit: 10 }, + }, { token: 'test-token', dispatcher: agent } ); @@ -652,8 +648,8 @@ describe('getEventsByCorrelationIdV4 over HTTP', () => { expect(query.get('limit')).toBe('10'); } - expect(result.events).toHaveLength(1); - expect(result.events[0].runId).toBe('wrun_1'); + expect(result.data).toHaveLength(1); + expect(result.data[0].runId).toBe('wrun_1'); agent.assertNoPendingInterceptors(); }); }); @@ -735,7 +731,7 @@ describe('v4 transport uses global fetch (observability)', () => { agent .get(origin) .intercept({ - path: '/api/v4/runs/wrun_1/events?returnAll=true', + path: '/api/v4/runs/wrun_1/events?remoteRefBehavior=resolve&returnAll=true', method: 'GET', }) .reply(200, encodeFrame({ _end: 1, hasMore: false }, new Uint8Array(0)), { @@ -747,8 +743,7 @@ describe('v4 transport uses global fetch (observability)', () => { const fetchSpy = vi.spyOn(globalThis, 'fetch'); await getWorkflowRunEventsV4( - 'wrun_1', - {}, + { runId: 'wrun_1' }, { token: 'test-token', dispatcher: agent } ); @@ -980,7 +975,7 @@ describe('createWorkflowRunEventV4 over HTTP', () => { agent .get(origin) .intercept({ - path: '/api/v4/runs/wrun_1/events?returnAll=true&cursor=eid%3Aevnt_1', + path: '/api/v4/runs/wrun_1/events?cursor=eid%3Aevnt_1&remoteRefBehavior=resolve&returnAll=true', method: 'GET', }) .reply( @@ -1121,7 +1116,7 @@ describe('createWorkflowRunEventV4 over HTTP', () => { agent .get(origin) .intercept({ - path: '/api/v4/runs/wrun_1/events?returnAll=true&cursor=eid%3Aevnt_1', + path: '/api/v4/runs/wrun_1/events?cursor=eid%3Aevnt_1&remoteRefBehavior=resolve&returnAll=true', method: 'GET', }) .reply( @@ -1804,7 +1799,7 @@ describe('v4 transport reports failures to the events recycler', () => { for (let i = 0; i < EVENTS_RECYCLE_AFTER_CONSECUTIVE_FAILURES; i++) { await expect( - getWorkflowRunEventsV4('wrun_1', {}, { token: 'test-token' }) + getWorkflowRunEventsV4({ runId: 'wrun_1' }, { token: 'test-token' }) ).rejects.toThrow(); // Still the same pool until the threshold is reached. if (i < EVENTS_RECYCLE_AFTER_CONSECUTIVE_FAILURES - 1) { diff --git a/packages/world-vercel/src/events-v4.ts b/packages/world-vercel/src/events-v4.ts index 44c62f23b8..3af42eec33 100644 --- a/packages/world-vercel/src/events-v4.ts +++ b/packages/world-vercel/src/events-v4.ts @@ -27,12 +27,13 @@ import { type Event, type EventResult, EventSchema, - type EventStreamObserver, type EventType, EventTypeSchema, getEventDataPayloadField, HookSchema, - type PaginationOptions, + type ListEventsByCorrelationIdParams, + type ListEventsParams, + type PaginatedResponse, StructuredErrorSchema, WaitSchema, WorkflowRunSchema, @@ -781,7 +782,7 @@ async function decodeCreateEventResponse( export async function createWorkflowRunStartedEventV4( input: CreateEventV4InputBase, config?: APIConfig, - onEvent?: EventStreamObserver + onEvent?: (event: Event) => void ) { const response = await postWorkflowRunEventV4( { ...input, eventType: 'run_started' }, @@ -1198,8 +1199,11 @@ async function postEventFrameOverWs( */ export type HookReceivedPreloadV4Result = /** The server streamed the replay log back as v4 frames. */ - | (ListEventsV4Result & { + | { kind: 'stream'; + events: Event[]; + cursor: string | null; + hasMore: boolean; /** * The canonical event this write created or converged on (the resume * claim winner's — ours or the producer's), named by the @@ -1208,7 +1212,7 @@ export type HookReceivedPreloadV4Result = canonicalEventId: string | undefined; /** Per-run event ceiling from the response header, when present. */ maxEvents: number | undefined; - }) + } /** * The server answered with the normal materialized CBOR body instead — * an older server, or one that declined the optimization. The @@ -1234,7 +1238,7 @@ export type HookReceivedPreloadV4Result = export async function createHookReceivedPreloadEventV4( input: CreateEventV4InputBase, config?: APIConfig, - onEvent?: EventStreamObserver + onEvent?: (event: Event) => void ): Promise { const response = await postWorkflowRunEventV4( { ...input, eventType: 'hook_received' }, @@ -1327,27 +1331,6 @@ export async function getEventV4( throw new Error(`v4 getEvent: empty frame stream for ${eventId}`); } -export interface ListEventsV4Params extends PaginationOptions { - /** - * Whether the backend resolves payload bytes into each frame body. - * `resolve` (default) streams the bytes; `lazy` emits empty-body frames - * (the ref descriptor stays in the frame meta) — for metadata-only - * listings that would otherwise download every payload just to discard - * it. - */ - remoteRefBehavior?: 'resolve' | 'lazy'; - /** Called synchronously after each event frame validates. */ - onEvent?: EventStreamObserver; -} - -export interface ListEventsV4Result { - events: Event[]; - /** Trailing event-log cursor, or null when the stream contained no events. */ - cursor: string | null; - /** Explicit "another page of results exists" flag from the sentinel. */ - hasMore: boolean; -} - class PartialEventStreamError extends WorkflowWorldError { constructor(message: string, cause?: unknown) { super(message, { code: 'TRANSPORT', cause }); @@ -1373,8 +1356,8 @@ async function consumeEventFrameStream( response: Response, opName: string, events: Event[], - onEvent?: EventStreamObserver -): Promise> { + onEvent?: (event: Event) => void +): Promise<{ cursor: string | null; hasMore: boolean }> { const contentType = response.headers.get('content-type'); if (!contentType?.startsWith(V4_FRAME_CONTENT_TYPE)) { throw new Error( @@ -1424,9 +1407,9 @@ async function consumeReplayLogResponse({ opName: string; events: Event[]; config?: APIConfig; - onEvent?: EventStreamObserver; -}): Promise> { - let page: Pick; + onEvent?: (event: Event) => void; +}): Promise<{ cursor: string | null; hasMore: boolean }> { + let page: { cursor: string | null; hasMore: boolean }; try { page = await consumeEventFrameStream(response, opName, events, onEvent); } catch (error) { @@ -1439,11 +1422,14 @@ async function consumeReplayLogResponse({ if (!page.hasMore) return page; assert(page.cursor, `v4 ${opName}: partial event stream missing cursor`); const suffix = await getWorkflowRunEventsV4( - runId, - { cursor: page.cursor, onEvent }, + { + runId, + pagination: { cursor: page.cursor }, + onEvent, + }, config ); - events.push(...suffix.events); + events.push(...suffix.data); return { cursor: suffix.cursor, hasMore: suffix.hasMore }; } @@ -1462,8 +1448,8 @@ async function consumeListFrameStream( config: APIConfig | undefined, opName: string, events: Event[], - onEvent?: EventStreamObserver -): Promise> { + onEvent?: (event: Event) => void +): Promise<{ cursor: string | null; hasMore: boolean }> { const response = await fetchV4( url, { method: 'GET', headers }, @@ -1478,20 +1464,26 @@ async function consumeListFrameStream( * Shared by the runId and correlationId list query builders so both send * `remoteRefBehavior` identically. */ -function appendListParams(sp: URLSearchParams, params: ListEventsV4Params) { - if (params.cursor) sp.set('cursor', params.cursor); - if (params.limit !== undefined) sp.set('limit', String(params.limit)); - if (params.sortOrder) sp.set('sortOrder', params.sortOrder); - if (params.remoteRefBehavior) { - sp.set('remoteRefBehavior', params.remoteRefBehavior); - } +function appendListParams( + sp: URLSearchParams, + params: ListEventsParams | ListEventsByCorrelationIdParams, + cursor = params.pagination?.cursor +) { + const { limit, sortOrder } = params.pagination ?? {}; + if (cursor) sp.set('cursor', cursor); + if (limit !== undefined) sp.set('limit', String(limit)); + if (sortOrder) sp.set('sortOrder', sortOrder); + sp.set( + 'remoteRefBehavior', + params.resolveData === 'none' ? 'lazy' : 'resolve' + ); } -function paginationToQuery(params: ListEventsV4Params): string { +function paginationToQuery(params: ListEventsParams, cursor?: string): string { const sp = new URLSearchParams(); // The World API uses an omitted limit for a complete event log. - if (params.limit === undefined) sp.set('returnAll', 'true'); - appendListParams(sp, params); + if (params.pagination?.limit === undefined) sp.set('returnAll', 'true'); + appendListParams(sp, params, cursor); return `?${sp.toString()}`; } @@ -1506,18 +1498,17 @@ function paginationToQuery(params: ListEventsV4Params): string { * after its last validated event instead of downloading accepted frames again. */ export async function getWorkflowRunEventsV4( - runId: string, - params: ListEventsV4Params = {}, + params: ListEventsParams, config?: APIConfig -): Promise { +): Promise> { const { baseUrl, headers } = await getHttpConfig(config); const events: Event[] = []; - let cursor = params.cursor; + let cursor = params.pagination?.cursor; while (true) { const url = - `${baseUrl}/v4/runs/${encodeURIComponent(runId)}/events` + - paginationToQuery({ ...params, cursor }); + `${baseUrl}/v4/runs/${encodeURIComponent(params.runId)}/events` + + paginationToQuery(params, cursor); try { const page = await consumeListFrameStream( url, @@ -1527,22 +1518,22 @@ export async function getWorkflowRunEventsV4( events, params.onEvent ); - if (params.limit === undefined && page.hasMore) { + if (params.pagination?.limit === undefined && page.hasMore) { if (!page.cursor || page.cursor === cursor) { throw new WorkflowWorldError( - `v4 listEvents: partial event stream made no cursor progress for run ${runId}`, + `v4 listEvents: partial event stream made no cursor progress for run ${params.runId}`, { code: 'SCHEMA_VALIDATION' } ); } cursor = page.cursor; continue; } - return { events, ...page }; + return { data: events, ...page }; } catch (error) { if (!(error instanceof PartialEventStreamError)) throw error; const lastEvent = events.at(-1); if ( - params.limit !== undefined || + params.pagination?.limit !== undefined || !lastEvent || `eid:${lastEvent.eventId}` === cursor ) { @@ -1568,15 +1559,13 @@ export async function getWorkflowRunEventsV4( * the page by run id. */ export async function getEventsByCorrelationIdV4( - correlationId: string, - runId: string, - params: ListEventsV4Params = {}, + params: ListEventsByCorrelationIdParams, config?: APIConfig -): Promise { +): Promise> { const { baseUrl, headers } = await getHttpConfig(config); const sp = new URLSearchParams(); - sp.set('correlationId', correlationId); - sp.set('runId', runId); + sp.set('correlationId', params.correlationId); + sp.set('runId', params.runId); appendListParams(sp, params); const url = `${baseUrl}/v4/events?${sp.toString()}`; const events: Event[] = []; @@ -1585,8 +1574,7 @@ export async function getEventsByCorrelationIdV4( headers, config, 'listEventsByCorrelationId', - events, - params.onEvent + events ); - return { events, ...page }; + return { data: events, ...page }; } diff --git a/packages/world-vercel/src/events.test.ts b/packages/world-vercel/src/events.test.ts index 95bad148ab..76d0c6896d 100644 --- a/packages/world-vercel/src/events.test.ts +++ b/packages/world-vercel/src/events.test.ts @@ -1887,7 +1887,7 @@ describe('createWorkflowRunEvent hook_received replay preload', () => { agent .get(ORIGIN) .intercept({ - path: '/api/v4/runs/wrun_1/events?returnAll=true&cursor=eid%3Aevnt_4', + path: '/api/v4/runs/wrun_1/events?cursor=eid%3Aevnt_4&remoteRefBehavior=resolve&returnAll=true', method: 'GET', }) .reply( diff --git a/packages/world-vercel/src/events.ts b/packages/world-vercel/src/events.ts index 4c114cf45f..49cffb774e 100644 --- a/packages/world-vercel/src/events.ts +++ b/packages/world-vercel/src/events.ts @@ -48,7 +48,6 @@ import { getEventDataPayloadField, isHookEventRequiringExistence, type ListEventsByCorrelationIdParams, - type ListEventsOptions, type ListEventsParams, type PaginatedResponse, validateUlidTimestamp, @@ -63,15 +62,10 @@ import { getEventsByCorrelationIdV4, getEventV4, getWorkflowRunEventsV4, - type ListEventsV4Params, } from './events-v4.js'; import { decode as decodeRunId } from './run-id/index.js'; import { cancelWorkflowRunV1, createWorkflowRunV1 } from './runs.js'; -import { - type APIConfig, - DEFAULT_RESOLVE_DATA_OPTION, - makeRequest, -} from './utils.js'; +import { type APIConfig, makeRequest } from './utils.js'; function validateWorkflowRunIdTimestamp(id: string): string | null { const raw = id.startsWith('wrun_') ? id.slice('wrun_'.length) : id; @@ -436,46 +430,18 @@ export async function getEvent( ); } -function getListEventsV4Params(params: ListEventsOptions): ListEventsV4Params { - const { pagination, resolveData = DEFAULT_RESOLVE_DATA_OPTION } = params; - // `resolveData: 'none'` leaves payload refs unresolved, so the backend can - // skip reading and streaming their contents. The validated lazy descriptors - // remain on the returned events. - return { - ...pagination, - remoteRefBehavior: resolveData === 'none' ? 'lazy' : 'resolve', - }; -} - export async function getWorkflowRunEvents( params: ListEventsParams, config?: APIConfig ): Promise> { - const result = await getWorkflowRunEventsV4( - params.runId, - { ...getListEventsV4Params(params), onEvent: params.onEvent }, - config - ); - - return { - data: result.events, - // The cursor is present even on the final page because it is also the - // incremental-load resume point. `hasMore` is the pagination signal. - cursor: result.cursor, - hasMore: result.hasMore, - }; + return getWorkflowRunEventsV4(params, config); } export async function getWorkflowRunEventsByCorrelationId( params: ListEventsByCorrelationIdParams, config?: APIConfig ): Promise> { - const result = await getEventsByCorrelationIdV4( - params.correlationId, - params.runId, - getListEventsV4Params(params), - config - ); + const result = await getEventsByCorrelationIdV4(params, config); // A correlation id is unique per run, not globally — a slot-numbered run // numbers its own steps, so `step_…001` names the first step of every such @@ -484,7 +450,7 @@ export async function getWorkflowRunEventsByCorrelationId( // `hasMore`/`cursor` stay the backend's, so a page that filters down to // nothing is still followed by the next one. return { - data: result.events.filter((event) => event.runId === params.runId), + data: result.data.filter((event) => event.runId === params.runId), // The cursor is present even on the final page because it is also the // incremental-load resume point. `hasMore` is the pagination signal. cursor: result.cursor, diff --git a/packages/world-vercel/src/trace-propagation.test.ts b/packages/world-vercel/src/trace-propagation.test.ts index b22b2d0ec6..a8155c6f5d 100644 --- a/packages/world-vercel/src/trace-propagation.test.ts +++ b/packages/world-vercel/src/trace-propagation.test.ts @@ -166,7 +166,7 @@ describe('v4 event requests (fetchV4) trace propagation', () => { agent .get(origin) .intercept({ - path: '/api/v4/runs/wrun_1/events?returnAll=true', + path: '/api/v4/runs/wrun_1/events?remoteRefBehavior=resolve&returnAll=true', method: 'GET', }) .reply(200, encodeFrame({ _end: 1, hasMore: false }, new Uint8Array(0)), { @@ -184,8 +184,7 @@ describe('v4 event requests (fetchV4) trace propagation', () => { traceId = span.spanContext().traceId; spanId = span.spanContext().spanId; await getWorkflowRunEventsV4( - 'wrun_1', - {}, + { runId: 'wrun_1' }, { token: 'test-token', dispatcher: agent } ); span.end(); diff --git a/packages/world/src/events.ts b/packages/world/src/events.ts index 5a1577e191..90279e298c 100644 --- a/packages/world/src/events.ts +++ b/packages/world/src/events.ts @@ -724,15 +724,6 @@ export type HookCreatedEventRequest = EventRequestOfType<'hook_created'>; export type HookReceivedEvent = z.infer; export type HookConflictEvent = z.infer; -/** - * Local observer invoked as each event is decoded from a streamed response. - * Worlds without a streaming implementation may ignore it. The callback runs - * synchronously, so its work intentionally applies response-stream backpressure. - * Keep it bounded to the event just decoded; large payloads may proportionally - * delay the next frame. - */ -export type EventStreamObserver = (event: Event) => void; - /** * Union of all possible event request types. * @internal Use CreateEventRequest or RunCreatedEventRequest instead. @@ -954,7 +945,7 @@ export interface CreateEventParams { * Observe replay-preload events as their frames are decoded. This is a * client-side delivery hook only; it is never serialized to a backend. */ - onEvent?: EventStreamObserver; + onEvent?: (event: Event) => void; } /** @@ -1137,19 +1128,19 @@ export interface GetEventParams { resolveData?: ResolveData; } -export interface ListEventsOptions { +export interface ListEventsParams { + runId: string; /** Omit `limit` to return every remaining event. */ pagination?: PaginationOptions; resolveData?: ResolveData; + /** + * Observe events as a streaming World decodes them. The callback runs + * synchronously and therefore applies response-stream backpressure. + */ + onEvent?: (event: Event) => void; } -export interface ListEventsParams extends ListEventsOptions { - runId: string; - /** Observe events as a streaming World decodes them. */ - onEvent?: EventStreamObserver; -} - -export interface ListEventsByCorrelationIdParams extends ListEventsOptions { +export interface ListEventsByCorrelationIdParams { correlationId: string; /** * The run the correlation id belongs to. A correlation id is unique per @@ -1160,4 +1151,7 @@ export interface ListEventsByCorrelationIdParams extends ListEventsOptions { * event id alone is not. */ runId: string; + /** Omit `limit` to return every remaining event. */ + pagination?: PaginationOptions; + resolveData?: ResolveData; } From 4d42010cf39e2827e2e3fba176fd3fa1d3f3dc8c Mon Sep 17 00:00:00 2001 From: Nathan Colosimo <110621881+NathanColosimo@users.noreply.github.com> Date: Fri, 14 Aug 2026 21:05:22 -0700 Subject: [PATCH 5/9] fix(world-vercel): preserve replay recovery cursor --- packages/world-vercel/src/events-v4.test.ts | 71 +++++++++++++++++++++ packages/world-vercel/src/events-v4.ts | 2 +- 2 files changed, 72 insertions(+), 1 deletion(-) diff --git a/packages/world-vercel/src/events-v4.test.ts b/packages/world-vercel/src/events-v4.test.ts index a40526ca4f..bdd5bc82b0 100644 --- a/packages/world-vercel/src/events-v4.test.ts +++ b/packages/world-vercel/src/events-v4.test.ts @@ -1013,6 +1013,77 @@ describe('createWorkflowRunEventV4 over HTTP', () => { agent.assertNoPendingInterceptors(); }); + it('preserves the POST cursor when truncation recovery returns an empty suffix', async () => { + const origin = + WORKFLOW_SERVER_URL_OVERRIDE || 'https://vercel-workflow.com'; + const agent = new MockAgent(); + agent.disableNetConnect(); + + agent + .get(origin) + .intercept({ + path: '/api/v4/runs/wrun_1/events/run_started', + method: 'POST', + headers: { accept: V4_FRAME_CONTENT_TYPE }, + }) + .reply( + 200, + Buffer.concat([ + encodeFrame( + { + eventId: 'evnt_1', + runId: 'wrun_1', + eventType: 'run_created', + createdAt: CREATED_AT, + eventData: { + deploymentId: 'dpl_1', + workflowName: 'workflow', + input: null, + }, + }, + new Uint8Array() + ), + encodeFrame( + { + eventId: 'evnt_2', + runId: 'wrun_1', + eventType: 'run_started', + createdAt: CREATED_AT, + }, + new Uint8Array() + ), + ]), + { + headers: { + 'content-type': V4_FRAME_CONTENT_TYPE, + 'x-wf-max-events': '10000', + }, + } + ); + agent + .get(origin) + .intercept({ + path: '/api/v4/runs/wrun_1/events?cursor=eid%3Aevnt_2&remoteRefBehavior=resolve&returnAll=true', + method: 'GET', + }) + .reply(200, encodeFrame({ _end: 1, hasMore: false }, new Uint8Array()), { + headers: { 'content-type': V4_FRAME_CONTENT_TYPE }, + }); + + const result = await createWorkflowRunStartedEventV4( + { runId: 'wrun_1', specVersion: 5 }, + { token: 'test-token', dispatcher: agent } + ); + + expect(result.events.map((event) => event.eventId)).toEqual([ + 'evnt_1', + 'evnt_2', + ]); + expect(result.cursor).toBe('eid:evnt_2'); + expect(result.hasMore).toBe(false); + agent.assertNoPendingInterceptors(); + }); + it('does not treat an event observer failure as stream truncation', async () => { const origin = WORKFLOW_SERVER_URL_OVERRIDE || 'https://vercel-workflow.com'; diff --git a/packages/world-vercel/src/events-v4.ts b/packages/world-vercel/src/events-v4.ts index 3af42eec33..38f30d865d 100644 --- a/packages/world-vercel/src/events-v4.ts +++ b/packages/world-vercel/src/events-v4.ts @@ -1430,7 +1430,7 @@ async function consumeReplayLogResponse({ config ); events.push(...suffix.data); - return { cursor: suffix.cursor, hasMore: suffix.hasMore }; + return { cursor: suffix.cursor ?? page.cursor, hasMore: suffix.hasMore }; } /** From b04bd9d5dcb9cf258206b6003569d46850636b9b Mon Sep 17 00:00:00 2001 From: Nathan Colosimo <110621881+NathanColosimo@users.noreply.github.com> Date: Mon, 17 Aug 2026 10:47:23 -0700 Subject: [PATCH 6/9] refactor(world-vercel): simplify replay frame recovery --- packages/world-vercel/src/events-v4.ts | 182 +++++++++++-------------- 1 file changed, 81 insertions(+), 101 deletions(-) diff --git a/packages/world-vercel/src/events-v4.ts b/packages/world-vercel/src/events-v4.ts index 38f30d865d..e21d0806b3 100644 --- a/packages/world-vercel/src/events-v4.ts +++ b/packages/world-vercel/src/events-v4.ts @@ -789,16 +789,12 @@ export async function createWorkflowRunStartedEventV4( 'event-stream', config ); - const events: Event[] = []; - const page = await consumeReplayLogResponse({ + const replay = await consumeReplayLogResponse( response, - runId: input.runId, - opName: 'createEvent', - events, - config, - onEvent, - }); - assert(page.cursor, 'v4 createEvent: event stream missing cursor'); + { runId: input.runId, onEvent }, + config + ); + assert(replay.cursor, 'v4 createEvent: event stream missing cursor'); const maxEvents = MaxEventsHeaderSchema.safeParse( response.headers.get(MAX_EVENTS_HEADER) ); @@ -809,7 +805,7 @@ export async function createWorkflowRunStartedEventV4( }); } - return { events, ...page, maxEvents: maxEvents.data }; + return { ...replay, maxEvents: maxEvents.data }; } /** One event of a v4 batch POST, index-aligned with the response results. */ @@ -1254,22 +1250,17 @@ export async function createHookReceivedPreloadEventV4( }; } - const events: Event[] = []; - const page = await consumeReplayLogResponse({ + const replay = await consumeReplayLogResponse( response, - runId: input.runId, - opName: 'createEvent', - events, - config, - onEvent, - }); + { runId: input.runId, onEvent }, + config + ); const maxEvents = MaxEventsHeaderSchema.safeParse( response.headers.get(MAX_EVENTS_HEADER) ); return { kind: 'stream', - events, - ...page, + ...replay, canonicalEventId: response.headers.get(EVENT_ID_HEADER) ?? undefined, maxEvents: maxEvents.success ? maxEvents.data : undefined, }; @@ -1337,21 +1328,6 @@ class PartialEventStreamError extends WorkflowWorldError { } } -async function* readEventFrames( - chunks: AsyncIterable, - opName: string, - events: readonly Event[] -): AsyncGenerator { - try { - yield* decodeFrames(chunks); - } catch (cause) { - throw new PartialEventStreamError( - `v4 ${opName}: event frame stream failed after ${events.length} events`, - cause - ); - } -} - async function consumeEventFrameStream( response: Response, opName: string, @@ -1366,20 +1342,35 @@ async function consumeEventFrameStream( } const chunks = response.body as unknown as AsyncIterable; + const frames = decodeFrames(chunks); - for await (const frame of readEventFrames(chunks, opName, events)) { - if (frame.meta._end === 1) { - const end = EventStreamEndSchema.parse(frame.meta); - return { cursor: end.next ?? null, hasMore: end.hasMore }; - } - if (Object.keys(frame.meta).some((key) => key.startsWith('_'))) { - throw new Error(`v4 ${opName}: unexpected control frame`); + try { + while (true) { + let next: IteratorResult; + try { + next = await frames.next(); + } catch (cause) { + throw new PartialEventStreamError( + `v4 ${opName}: event frame stream failed after ${events.length} events`, + cause + ); + } + if (next.done) break; + + const frame = next.value; + if (frame.meta._end === 1) { + const end = EventStreamEndSchema.parse(frame.meta); + return { cursor: end.next ?? null, hasMore: end.hasMore }; + } + if (Object.keys(frame.meta).some((key) => key.startsWith('_'))) { + throw new Error(`v4 ${opName}: unexpected control frame`); + } + const event = decodeEventFrame(frame); + events.push(event); + onEvent?.(event); } - const event = decodeEventFrame(frame); - events.push(event); - // Observer/application failures remain outside the frame reader, so they - // are not mistaken for truncation or skipped by partial-stream recovery. - onEvent?.(event); + } finally { + void frames.return(undefined); } throw new PartialEventStreamError( @@ -1394,24 +1385,24 @@ async function consumeEventFrameStream( * continuation instead of making run_started download its accepted prefix * again. */ -async function consumeReplayLogResponse({ - response, - runId, - opName, - events, - config, - onEvent, -}: { - response: Response; - runId: string; - opName: string; +async function consumeReplayLogResponse( + response: Response, + params: Pick, + config?: APIConfig +): Promise<{ events: Event[]; - config?: APIConfig; - onEvent?: (event: Event) => void; -}): Promise<{ cursor: string | null; hasMore: boolean }> { + cursor: string | null; + hasMore: boolean; +}> { + const events: Event[] = []; let page: { cursor: string | null; hasMore: boolean }; try { - page = await consumeEventFrameStream(response, opName, events, onEvent); + page = await consumeEventFrameStream( + response, + 'createEvent', + events, + params.onEvent + ); } catch (error) { if (!(error instanceof PartialEventStreamError)) throw error; const lastEvent = events.at(-1); @@ -1419,44 +1410,22 @@ async function consumeReplayLogResponse({ page = { cursor: `eid:${lastEvent.eventId}`, hasMore: true }; } - if (!page.hasMore) return page; - assert(page.cursor, `v4 ${opName}: partial event stream missing cursor`); + if (!page.hasMore) return { events, ...page }; + assert(page.cursor, 'v4 createEvent: partial event stream missing cursor'); const suffix = await getWorkflowRunEventsV4( { - runId, + runId: params.runId, pagination: { cursor: page.cursor }, - onEvent, + onEvent: params.onEvent, }, config ); events.push(...suffix.data); - return { cursor: suffix.cursor ?? page.cursor, hasMore: suffix.hasMore }; -} - -/** - * Drive a v4 frame-stream list response into an in-memory page. Used by - * both the by-runId and by-correlationId list endpoints — the wire - * shape is identical, only the URL differs. - * - * `headers` come from the caller's single getHttpConfig resolution (the - * same call that produced the baseUrl in `url`) so each LIST resolves - * auth exactly once. - */ -async function consumeListFrameStream( - url: string, - headers: Headers, - config: APIConfig | undefined, - opName: string, - events: Event[], - onEvent?: (event: Event) => void -): Promise<{ cursor: string | null; hasMore: boolean }> { - const response = await fetchV4( - url, - { method: 'GET', headers }, - config, - opName - ); - return consumeEventFrameStream(response, opName, events, onEvent); + return { + events, + cursor: suffix.cursor ?? page.cursor, + hasMore: suffix.hasMore, + }; } /** @@ -1467,7 +1436,7 @@ async function consumeListFrameStream( function appendListParams( sp: URLSearchParams, params: ListEventsParams | ListEventsByCorrelationIdParams, - cursor = params.pagination?.cursor + cursor: string | null ) { const { limit, sortOrder } = params.pagination ?? {}; if (cursor) sp.set('cursor', cursor); @@ -1479,7 +1448,10 @@ function appendListParams( ); } -function paginationToQuery(params: ListEventsParams, cursor?: string): string { +function paginationToQuery( + params: ListEventsParams, + cursor: string | null +): string { const sp = new URLSearchParams(); // The World API uses an omitted limit for a complete event log. if (params.pagination?.limit === undefined) sp.set('returnAll', 'true'); @@ -1503,17 +1475,21 @@ export async function getWorkflowRunEventsV4( ): Promise> { const { baseUrl, headers } = await getHttpConfig(config); const events: Event[] = []; - let cursor = params.pagination?.cursor; + let cursor = params.pagination?.cursor ?? null; while (true) { const url = `${baseUrl}/v4/runs/${encodeURIComponent(params.runId)}/events` + paginationToQuery(params, cursor); try { - const page = await consumeListFrameStream( + const response = await fetchV4( url, - headers, + { method: 'GET', headers }, config, + 'listEvents' + ); + const page = await consumeEventFrameStream( + response, 'listEvents', events, params.onEvent @@ -1566,13 +1542,17 @@ export async function getEventsByCorrelationIdV4( const sp = new URLSearchParams(); sp.set('correlationId', params.correlationId); sp.set('runId', params.runId); - appendListParams(sp, params); + appendListParams(sp, params, params.pagination?.cursor ?? null); const url = `${baseUrl}/v4/events?${sp.toString()}`; const events: Event[] = []; - const page = await consumeListFrameStream( + const response = await fetchV4( url, - headers, + { method: 'GET', headers }, config, + 'listEventsByCorrelationId' + ); + const page = await consumeEventFrameStream( + response, 'listEventsByCorrelationId', events ); From 8984a757fe00c02557253e90d39d92a8f2ea63e7 Mon Sep 17 00:00:00 2001 From: Nathan Colosimo <110621881+NathanColosimo@users.noreply.github.com> Date: Tue, 18 Aug 2026 14:11:36 -0700 Subject: [PATCH 7/9] fix(world-vercel): bound replay stream recovery --- packages/world-vercel/src/events-v4.test.ts | 231 ++++++++++---------- packages/world-vercel/src/events-v4.ts | 225 ++++++++++++------- packages/world-vercel/src/events.test.ts | 41 +++- packages/world-vercel/src/events.ts | 51 +++-- packages/world-vercel/src/http-core.ts | 7 +- 5 files changed, 328 insertions(+), 227 deletions(-) diff --git a/packages/world-vercel/src/events-v4.test.ts b/packages/world-vercel/src/events-v4.test.ts index bdd5bc82b0..a9e966efe1 100644 --- a/packages/world-vercel/src/events-v4.test.ts +++ b/packages/world-vercel/src/events-v4.test.ts @@ -29,6 +29,13 @@ import { import { WORKFLOW_SERVER_URL_OVERRIDE } from './utils.js'; const CREATED_AT = '2026-06-10T00:00:00.000Z'; +const ORIGIN = WORKFLOW_SERVER_URL_OVERRIDE || 'https://vercel-workflow.com'; + +function mockAgent() { + const agent = new MockAgent(); + agent.disableNetConnect(); + return agent; +} function createEventBody( event: AnyEventRequest, @@ -264,10 +271,7 @@ describe('throwForErrorResponse', () => { */ describe('getWorkflowRunEventsV4 over HTTP', () => { it('parses a frame stream fetched via a custom dispatcher', async () => { - const origin = - WORKFLOW_SERVER_URL_OVERRIDE || 'https://vercel-workflow.com'; - const agent = new MockAgent(); - agent.disableNetConnect(); + const agent = mockAgent(); const body = new TextEncoder().encode('payload-bytes'); const frames = Buffer.concat([ @@ -292,7 +296,7 @@ describe('getWorkflowRunEventsV4 over HTTP', () => { ]); agent - .get(origin) + .get(ORIGIN) .intercept({ path: '/api/v4/runs/wrun_1/events?remoteRefBehavior=resolve&returnAll=true', method: 'GET', @@ -316,13 +320,10 @@ describe('getWorkflowRunEventsV4 over HTTP', () => { }); it('does not resume past a GET observer failure', async () => { - const origin = - WORKFLOW_SERVER_URL_OVERRIDE || 'https://vercel-workflow.com'; - const agent = new MockAgent(); - agent.disableNetConnect(); + const agent = mockAgent(); agent - .get(origin) + .get(ORIGIN) .intercept({ path: '/api/v4/runs/wrun_1/events?remoteRefBehavior=resolve&returnAll=true', method: 'GET', @@ -368,13 +369,10 @@ describe('getWorkflowRunEventsV4 over HTTP', () => { ['an unknown event type', { eventType: 'future_event', eventData: {} }], ['invalid event metadata', { eventType: 'run_created', eventData: {} }], ])('rejects %s', async (_description, meta) => { - const origin = - WORKFLOW_SERVER_URL_OVERRIDE || 'https://vercel-workflow.com'; - const agent = new MockAgent(); - agent.disableNetConnect(); + const agent = mockAgent(); agent - .get(origin) + .get(ORIGIN) .intercept({ path: '/api/v4/runs/wrun_1/events?remoteRefBehavior=resolve&returnAll=true', method: 'GET', @@ -398,10 +396,7 @@ describe('getWorkflowRunEventsV4 over HTTP', () => { }); it('captures an explicit hasMore from the sentinel, independent of next', async () => { - const origin = - WORKFLOW_SERVER_URL_OVERRIDE || 'https://vercel-workflow.com'; - const agent = new MockAgent(); - agent.disableNetConnect(); + const agent = mockAgent(); // The regression shape: a final page still carries a trailing `next` // cursor (incremental-load resume point) but hasMore is false. @@ -427,7 +422,7 @@ describe('getWorkflowRunEventsV4 over HTTP', () => { ]); agent - .get(origin) + .get(ORIGIN) .intercept({ path: '/api/v4/runs/wrun_1/events?remoteRefBehavior=resolve&returnAll=true', method: 'GET', @@ -446,10 +441,7 @@ describe('getWorkflowRunEventsV4 over HTTP', () => { }); it('rejects an end frame without hasMore', async () => { - const origin = - WORKFLOW_SERVER_URL_OVERRIDE || 'https://vercel-workflow.com'; - const agent = new MockAgent(); - agent.disableNetConnect(); + const agent = mockAgent(); const frames = encodeFrame( { _end: 1, next: 'cursor-2' }, @@ -457,7 +449,7 @@ describe('getWorkflowRunEventsV4 over HTTP', () => { ); agent - .get(origin) + .get(ORIGIN) .intercept({ path: '/api/v4/runs/wrun_1/events?remoteRefBehavior=resolve&returnAll=true', method: 'GET', @@ -475,10 +467,7 @@ describe('getWorkflowRunEventsV4 over HTTP', () => { }); it('throws when the stream ends without the end sentinel (truncated response)', async () => { - const origin = - WORKFLOW_SERVER_URL_OVERRIDE || 'https://vercel-workflow.com'; - const agent = new MockAgent(); - agent.disableNetConnect(); + const agent = mockAgent(); // A complete event frame but NO `{_end: 1}` sentinel — what a response // truncated on a frame boundary looks like. Returning this as a @@ -499,7 +488,7 @@ describe('getWorkflowRunEventsV4 over HTTP', () => { ); agent - .get(origin) + .get(ORIGIN) .intercept({ path: '/api/v4/runs/wrun_1/events?limit=500&remoteRefBehavior=resolve', method: 'GET', @@ -517,13 +506,10 @@ describe('getWorkflowRunEventsV4 over HTTP', () => { }); it('resumes a truncated full stream after its last accepted event', async () => { - const origin = - WORKFLOW_SERVER_URL_OVERRIDE || 'https://vercel-workflow.com'; - const agent = new MockAgent(); - agent.disableNetConnect(); + const agent = mockAgent(); agent - .get(origin) + .get(ORIGIN) .intercept({ path: '/api/v4/runs/wrun_1/events?remoteRefBehavior=resolve&returnAll=true', method: 'GET', @@ -547,7 +533,7 @@ describe('getWorkflowRunEventsV4 over HTTP', () => { { headers: { 'content-type': V4_FRAME_CONTENT_TYPE } } ); agent - .get(origin) + .get(ORIGIN) .intercept({ path: '/api/v4/runs/wrun_1/events?cursor=eid%3Aevnt_1&remoteRefBehavior=resolve&returnAll=true', method: 'GET', @@ -585,6 +571,74 @@ describe('getWorkflowRunEventsV4 over HTTP', () => { expect(result.hasMore).toBe(false); agent.assertNoPendingInterceptors(); }); + + it('stops after three partial-stream recovery retries', async () => { + const agent = mockAgent(); + const paths = [ + '/api/v4/runs/wrun_1/events?remoteRefBehavior=resolve&returnAll=true', + '/api/v4/runs/wrun_1/events?cursor=eid%3Aevnt_1&remoteRefBehavior=resolve&returnAll=true', + '/api/v4/runs/wrun_1/events?cursor=eid%3Aevnt_2&remoteRefBehavior=resolve&returnAll=true', + '/api/v4/runs/wrun_1/events?cursor=eid%3Aevnt_3&remoteRefBehavior=resolve&returnAll=true', + ]; + + for (const [index, path] of paths.entries()) { + agent + .get(ORIGIN) + .intercept({ path, method: 'GET' }) + .reply( + 200, + encodeFrame( + { + eventId: `evnt_${index + 1}`, + runId: 'wrun_1', + eventType: 'run_created', + createdAt: CREATED_AT, + eventData: { + deploymentId: 'dpl_1', + workflowName: 'workflow', + input: null, + }, + }, + new Uint8Array() + ), + { headers: { 'content-type': V4_FRAME_CONTENT_TYPE } } + ); + } + + await expect( + getWorkflowRunEventsV4( + { runId: 'wrun_1' }, + { token: 'test-token', dispatcher: agent } + ) + ).rejects.toThrow(/end-of-stream sentinel/); + agent.assertNoPendingInterceptors(); + }); + + it('rejects an unexpected clean pagination response', async () => { + const agent = mockAgent(); + agent + .get(ORIGIN) + .intercept({ + path: '/api/v4/runs/wrun_1/events?remoteRefBehavior=resolve&returnAll=true', + method: 'GET', + }) + .reply( + 200, + encodeFrame( + { _end: 1, next: 'eid:evnt_1', hasMore: true }, + new Uint8Array() + ), + { headers: { 'content-type': V4_FRAME_CONTENT_TYPE } } + ); + + await expect( + getWorkflowRunEventsV4( + { runId: 'wrun_1' }, + { token: 'test-token', dispatcher: agent } + ) + ).rejects.toThrow(/returnAll response was unexpectedly paginated/); + agent.assertNoPendingInterceptors(); + }); }); /** @@ -936,14 +990,11 @@ describe('createWorkflowRunEventV4 over HTTP', () => { }); it('continues a truncated run_started stream after its last event', async () => { - const origin = - WORKFLOW_SERVER_URL_OVERRIDE || 'https://vercel-workflow.com'; - const agent = new MockAgent(); - agent.disableNetConnect(); + const agent = mockAgent(); const observed: string[] = []; agent - .get(origin) + .get(ORIGIN) .intercept({ path: '/api/v4/runs/wrun_1/events/run_started', method: 'POST', @@ -973,7 +1024,7 @@ describe('createWorkflowRunEventV4 over HTTP', () => { } ); agent - .get(origin) + .get(ORIGIN) .intercept({ path: '/api/v4/runs/wrun_1/events?cursor=eid%3Aevnt_1&remoteRefBehavior=resolve&returnAll=true', method: 'GET', @@ -1014,13 +1065,10 @@ describe('createWorkflowRunEventV4 over HTTP', () => { }); it('preserves the POST cursor when truncation recovery returns an empty suffix', async () => { - const origin = - WORKFLOW_SERVER_URL_OVERRIDE || 'https://vercel-workflow.com'; - const agent = new MockAgent(); - agent.disableNetConnect(); + const agent = mockAgent(); agent - .get(origin) + .get(ORIGIN) .intercept({ path: '/api/v4/runs/wrun_1/events/run_started', method: 'POST', @@ -1061,7 +1109,7 @@ describe('createWorkflowRunEventV4 over HTTP', () => { } ); agent - .get(origin) + .get(ORIGIN) .intercept({ path: '/api/v4/runs/wrun_1/events?cursor=eid%3Aevnt_2&remoteRefBehavior=resolve&returnAll=true', method: 'GET', @@ -1084,72 +1132,11 @@ describe('createWorkflowRunEventV4 over HTTP', () => { agent.assertNoPendingInterceptors(); }); - it('does not treat an event observer failure as stream truncation', async () => { - const origin = - WORKFLOW_SERVER_URL_OVERRIDE || 'https://vercel-workflow.com'; - const agent = new MockAgent(); - agent.disableNetConnect(); - - agent - .get(origin) - .intercept({ - path: '/api/v4/runs/wrun_1/events/run_started', - method: 'POST', - headers: { accept: V4_FRAME_CONTENT_TYPE }, - }) - .reply( - 200, - Buffer.concat([ - encodeFrame( - { - eventId: 'evnt_1', - runId: 'wrun_1', - eventType: 'run_created', - createdAt: CREATED_AT, - eventData: { - deploymentId: 'dpl_1', - workflowName: 'workflow', - input: null, - }, - }, - new Uint8Array() - ), - encodeFrame( - { _end: 1, next: 'eid:evnt_1', hasMore: false }, - new Uint8Array() - ), - ]), - { - headers: { - 'content-type': V4_FRAME_CONTENT_TYPE, - 'x-wf-max-events': '10000', - }, - } - ); - - const observerError = new WorkflowWorldError('observer failed', { - code: 'TRANSPORT', - }); - await expect( - createWorkflowRunStartedEventV4( - { runId: 'wrun_1', specVersion: 5 }, - { token: 'test-token', dispatcher: agent }, - () => { - throw observerError; - } - ) - ).rejects.toBe(observerError); - agent.assertNoPendingInterceptors(); - }); - it('continues a graceful partial run_started stream from its sentinel cursor', async () => { - const origin = - WORKFLOW_SERVER_URL_OVERRIDE || 'https://vercel-workflow.com'; - const agent = new MockAgent(); - agent.disableNetConnect(); + const agent = mockAgent(); agent - .get(origin) + .get(ORIGIN) .intercept({ path: '/api/v4/runs/wrun_1/events/run_started', method: 'POST', @@ -1185,7 +1172,7 @@ describe('createWorkflowRunEventV4 over HTTP', () => { } ); agent - .get(origin) + .get(ORIGIN) .intercept({ path: '/api/v4/runs/wrun_1/events?cursor=eid%3Aevnt_1&remoteRefBehavior=resolve&returnAll=true', method: 'GET', @@ -1834,10 +1821,9 @@ describe('v4 POST frame meta forwards every field the splitter produces', () => /** * The recycler in http-client only sees transport failures the v4 client - * reports to it. This covers that wiring end to end: a `fetch()` that rejects - * the way a wedged HTTP/2 session does must retire the shared events pool once - * the failures reach the threshold. Without the `onTransportOutcome` hook in - * `fetchV4` the recycler is never told anything and the pool lives forever. + * reports to it. A streamed response resolves `fetch()` as soon as headers + * arrive, before its body can fail, so the body consumer must own the success + * report or that early success erases every later stream failure. */ describe('v4 transport reports failures to the events recycler', () => { // There is only an undici pool to retire while the adapter owns one: @@ -1861,8 +1847,17 @@ describe('v4 transport reports failures to the events recycler', () => { }), }); - it('rebuilds the shared pool after repeated stream timeouts', async () => { - vi.spyOn(globalThis, 'fetch').mockRejectedValue(wedgedSessionError()); + it('rebuilds the shared pool after repeated response-body timeouts', async () => { + vi.spyOn(globalThis, 'fetch').mockImplementation(async () => { + const body = new ReadableStream({ + start(controller) { + controller.error(wedgedSessionError()); + }, + }); + return new Response(body, { + headers: { 'content-type': V4_FRAME_CONTENT_TYPE }, + }); + }); // No `dispatcher` in the config: the request must resolve the shared one, // which is what the recycler owns. diff --git a/packages/world-vercel/src/events-v4.ts b/packages/world-vercel/src/events-v4.ts index e21d0806b3..cd10aa3eba 100644 --- a/packages/world-vercel/src/events-v4.ts +++ b/packages/world-vercel/src/events-v4.ts @@ -100,15 +100,23 @@ import { isWsEventsTransportEnabled } from './ws-transport-enabled.js'; * for a large run can legitimately take a while to drain — a whole-request * deadline would abort it mid-stream. */ +interface V4Response { + response: Response; + reportTransportOutcome(error?: unknown): void; +} + async function fetchV4( url: string, init: { method: string; headers: Headers; body?: Uint8Array }, config: APIConfig | undefined, opName: string, + transportSuccess: 'headers' | 'body', attributes?: Record -): Promise { +): Promise { const dispatcher = getEventsDispatcher(config); - return instrumentedFetch({ + const reportTransportOutcome = (error?: unknown) => + noteEventsTransportOutcome(dispatcher, error); + const response = await instrumentedFetch({ method: init.method, url, headers: init.headers, @@ -119,8 +127,8 @@ async function fetchV4( // request builds a fresh one. undici keeps a black-holed HTTP/2 session in // service indefinitely, so without this every request routed onto it fails // until the compute instance is recycled — see noteEventsTransportOutcome. - onTransportOutcome: (error) => - noteEventsTransportOutcome(dispatcher, error), + onTransportOutcome: reportTransportOutcome, + deferTransportSuccess: transportSuccess === 'body', timeoutMs: null, logLabel: opName, // Read the body as bytes, not text: a CBOR error body (the fence 412 @@ -134,6 +142,7 @@ async function fetchV4( url ), }); + return { response, reportTransportOutcome }; } const EVENT_ID_HEADER = 'x-wf-event-id'; @@ -712,6 +721,7 @@ async function postWorkflowRunEventV4( { method: 'POST', headers, body: frame }, config, 'createEvent', + responseType === 'event-stream' ? 'body' : 'headers', { ...WorkflowEventsTransport('http'), ...WorkflowEventType(input.eventType), @@ -741,7 +751,11 @@ export async function createWorkflowRunEventV4( if (reply) return decodeCreateEventResponse(reply, input.eventType); } - const response = await postWorkflowRunEventV4(input, 'materialized', config); + const { response } = await postWorkflowRunEventV4( + input, + 'materialized', + config + ); const contentType = response.headers.get('content-type'); if (contentType?.startsWith(V4_FRAME_CONTENT_TYPE)) { @@ -784,19 +798,19 @@ export async function createWorkflowRunStartedEventV4( config?: APIConfig, onEvent?: (event: Event) => void ) { - const response = await postWorkflowRunEventV4( + const stream = await postWorkflowRunEventV4( { ...input, eventType: 'run_started' }, 'event-stream', config ); const replay = await consumeReplayLogResponse( - response, + stream, { runId: input.runId, onEvent }, config ); assert(replay.cursor, 'v4 createEvent: event stream missing cursor'); const maxEvents = MaxEventsHeaderSchema.safeParse( - response.headers.get(MAX_EVENTS_HEADER) + stream.response.headers.get(MAX_EVENTS_HEADER) ); if (!maxEvents.success) { throw new WorkflowWorldError('v4 createEvent: invalid max-events header', { @@ -893,11 +907,12 @@ export async function createWorkflowRunEventsBatchV4( // span carries only wire-level facts. workflow.event.type is deliberately // absent — it names a single event write, and tagging a batch with its // first event's type misclassifies the traffic. - const response = await fetchV4( + const { response } = await fetchV4( url, { method: 'POST', headers, body }, config, 'createEventBatch', + 'headers', { ...WorkflowEventsTransport('http'), 'workflow.batch.bytes': body.byteLength, @@ -1189,17 +1204,20 @@ async function postEventFrameOverWs( ); } +export interface ReplayLogResult { + events: Event[]; + cursor: string | null; + hasMore: boolean; +} + /** * Result of a `hook_received` POST that opted into the replay-log preload, * discriminated on `kind` (keyed on the response content type). */ export type HookReceivedPreloadV4Result = /** The server streamed the replay log back as v4 frames. */ - | { + | (ReplayLogResult & { kind: 'stream'; - events: Event[]; - cursor: string | null; - hasMore: boolean; /** * The canonical event this write created or converged on (the resume * claim winner's — ours or the producer's), named by the @@ -1208,7 +1226,7 @@ export type HookReceivedPreloadV4Result = canonicalEventId: string | undefined; /** Per-run event ceiling from the response header, when present. */ maxEvents: number | undefined; - } + }) /** * The server answered with the normal materialized CBOR body instead — * an older server, or one that declined the optimization. The @@ -1236,14 +1254,16 @@ export async function createHookReceivedPreloadEventV4( config?: APIConfig, onEvent?: (event: Event) => void ): Promise { - const response = await postWorkflowRunEventV4( + const stream = await postWorkflowRunEventV4( { ...input, eventType: 'hook_received' }, 'event-stream', config ); + const { response } = stream; const contentType = response.headers.get('content-type'); if (!contentType?.startsWith(V4_FRAME_CONTENT_TYPE)) { + stream.reportTransportOutcome(); return { kind: 'materialized', result: await decodeCreateEventResponse(response, 'hook_received'), @@ -1251,7 +1271,7 @@ export async function createHookReceivedPreloadEventV4( } const replay = await consumeReplayLogResponse( - response, + stream, { runId: input.runId, onEvent }, config ); @@ -1293,14 +1313,17 @@ export async function getEventV4( const url = `${baseUrl}/v4/runs/${encodeURIComponent(runId)}/events/${encodeURIComponent(eventId)}` + `?remoteRefBehavior=${remoteRefBehavior}`; - const response = await fetchV4( + const stream = await fetchV4( url, { method: 'GET', headers }, config, - 'getEvent' + 'getEvent', + 'body' ); + const { response } = stream; const contentType = response.headers.get('content-type'); if (!contentType?.startsWith(V4_FRAME_CONTENT_TYPE)) { + stream.reportTransportOutcome(); throw new Error( `v4 getEvent: expected ${V4_FRAME_CONTENT_TYPE}, got ${contentType ?? '(none)'}` ); @@ -1316,8 +1339,15 @@ export async function getEventV4( // GET emits a single frame (no sentinel); decodeFrames returns at EOF // after yielding it. - for await (const frame of decodeFrames(chunks)) { - return decodeEventFrame(frame); + try { + for await (const frame of decodeFrames(chunks)) { + stream.reportTransportOutcome(); + return decodeEventFrame(frame); + } + stream.reportTransportOutcome(); + } catch (error) { + stream.reportTransportOutcome(error); + throw error; } throw new Error(`v4 getEvent: empty frame stream for ${eventId}`); } @@ -1328,14 +1358,29 @@ class PartialEventStreamError extends WorkflowWorldError { } } +const MAX_PARTIAL_EVENT_STREAM_RETRIES = 3; + +type EventFrameStreamResult = + | { + kind: 'complete'; + cursor: string | null; + hasMore: boolean; + } + | { + kind: 'partial'; + error: PartialEventStreamError; + }; + async function consumeEventFrameStream( - response: Response, + stream: V4Response, opName: string, events: Event[], onEvent?: (event: Event) => void -): Promise<{ cursor: string | null; hasMore: boolean }> { +): Promise { + const { response } = stream; const contentType = response.headers.get('content-type'); if (!contentType?.startsWith(V4_FRAME_CONTENT_TYPE)) { + stream.reportTransportOutcome(); throw new Error( `v4 ${opName}: expected ${V4_FRAME_CONTENT_TYPE}, got ${contentType ?? '(none)'}` ); @@ -1350,33 +1395,47 @@ async function consumeEventFrameStream( try { next = await frames.next(); } catch (cause) { - throw new PartialEventStreamError( + const error = new PartialEventStreamError( `v4 ${opName}: event frame stream failed after ${events.length} events`, cause ); + stream.reportTransportOutcome(error); + return { kind: 'partial', error }; } if (next.done) break; - const frame = next.value; - if (frame.meta._end === 1) { - const end = EventStreamEndSchema.parse(frame.meta); - return { cursor: end.next ?? null, hasMore: end.hasMore }; - } - if (Object.keys(frame.meta).some((key) => key.startsWith('_'))) { - throw new Error(`v4 ${opName}: unexpected control frame`); + try { + const frame = next.value; + if (frame.meta._end === 1) { + const end = EventStreamEndSchema.parse(frame.meta); + stream.reportTransportOutcome(); + return { + kind: 'complete', + cursor: end.next ?? null, + hasMore: end.hasMore, + }; + } + if (Object.keys(frame.meta).some((key) => key.startsWith('_'))) { + throw new Error(`v4 ${opName}: unexpected control frame`); + } + const event = decodeEventFrame(frame); + events.push(event); + onEvent?.(event); + } catch (error) { + stream.reportTransportOutcome(); + throw error; } - const event = decodeEventFrame(frame); - events.push(event); - onEvent?.(event); } } finally { void frames.return(undefined); } - throw new PartialEventStreamError( + const error = new PartialEventStreamError( `v4 ${opName}: frame stream ended without the end-of-stream sentinel ` + `(${events.length} events read) — truncated response?` ); + stream.reportTransportOutcome(error); + return { kind: 'partial', error }; } /** @@ -1386,28 +1445,24 @@ async function consumeEventFrameStream( * again. */ async function consumeReplayLogResponse( - response: Response, + stream: V4Response, params: Pick, config?: APIConfig -): Promise<{ - events: Event[]; - cursor: string | null; - hasMore: boolean; -}> { +): Promise { const events: Event[] = []; let page: { cursor: string | null; hasMore: boolean }; - try { - page = await consumeEventFrameStream( - response, - 'createEvent', - events, - params.onEvent - ); - } catch (error) { - if (!(error instanceof PartialEventStreamError)) throw error; + const consumed = await consumeEventFrameStream( + stream, + 'createEvent', + events, + params.onEvent + ); + if (consumed.kind === 'partial') { const lastEvent = events.at(-1); - if (!lastEvent) throw error; + if (!lastEvent) throw consumed.error; page = { cursor: `eid:${lastEvent.eventId}`, hasMore: true }; + } else { + page = consumed; } if (!page.hasMore) return { events, ...page }; @@ -1421,6 +1476,10 @@ async function consumeReplayLogResponse( config ); events.push(...suffix.data); + assert( + suffix.data.length === 0 || suffix.cursor, + 'v4 createEvent: non-empty continuation missing cursor' + ); return { events, cursor: suffix.cursor ?? page.cursor, @@ -1477,46 +1536,44 @@ export async function getWorkflowRunEventsV4( const events: Event[] = []; let cursor = params.pagination?.cursor ?? null; - while (true) { + for (let partialRetries = 0; ; partialRetries++) { const url = `${baseUrl}/v4/runs/${encodeURIComponent(params.runId)}/events` + paginationToQuery(params, cursor); - try { - const response = await fetchV4( - url, - { method: 'GET', headers }, - config, - 'listEvents' - ); - const page = await consumeEventFrameStream( - response, - 'listEvents', - events, - params.onEvent - ); - if (params.pagination?.limit === undefined && page.hasMore) { - if (!page.cursor || page.cursor === cursor) { - throw new WorkflowWorldError( - `v4 listEvents: partial event stream made no cursor progress for run ${params.runId}`, - { code: 'SCHEMA_VALIDATION' } - ); - } - cursor = page.cursor; - continue; - } - return { data: events, ...page }; - } catch (error) { - if (!(error instanceof PartialEventStreamError)) throw error; + const stream = await fetchV4( + url, + { method: 'GET', headers }, + config, + 'listEvents', + 'body' + ); + const result = await consumeEventFrameStream( + stream, + 'listEvents', + events, + params.onEvent + ); + if (result.kind === 'partial') { const lastEvent = events.at(-1); if ( + partialRetries === MAX_PARTIAL_EVENT_STREAM_RETRIES || params.pagination?.limit !== undefined || !lastEvent || `eid:${lastEvent.eventId}` === cursor ) { - throw error; + throw result.error; } cursor = `eid:${lastEvent.eventId}`; + continue; } + + if (params.pagination?.limit === undefined && result.hasMore) { + throw new WorkflowWorldError( + `v4 listEvents: returnAll response was unexpectedly paginated for run ${params.runId}`, + { code: 'SCHEMA_VALIDATION' } + ); + } + return { data: events, cursor: result.cursor, hasMore: result.hasMore }; } } @@ -1545,16 +1602,18 @@ export async function getEventsByCorrelationIdV4( appendListParams(sp, params, params.pagination?.cursor ?? null); const url = `${baseUrl}/v4/events?${sp.toString()}`; const events: Event[] = []; - const response = await fetchV4( + const stream = await fetchV4( url, { method: 'GET', headers }, config, - 'listEventsByCorrelationId' + 'listEventsByCorrelationId', + 'body' ); - const page = await consumeEventFrameStream( - response, + const result = await consumeEventFrameStream( + stream, 'listEventsByCorrelationId', events ); - return { data: events, ...page }; + if (result.kind === 'partial') throw result.error; + return { data: events, cursor: result.cursor, hasMore: result.hasMore }; } diff --git a/packages/world-vercel/src/events.test.ts b/packages/world-vercel/src/events.test.ts index 76d0c6896d..87ba906319 100644 --- a/packages/world-vercel/src/events.test.ts +++ b/packages/world-vercel/src/events.test.ts @@ -1,10 +1,11 @@ import { Buffer } from 'node:buffer'; import { gzipSync } from 'node:zlib'; +import { WorkflowWorldError } from '@workflow/errors'; import type { AnyEventRequest, CreateEventParams } from '@workflow/world'; import { decode, encode } from 'cbor-x'; import { ulid } from 'ulid'; import { MockAgent } from 'undici'; -import { describe, expect, it } from 'vitest'; +import { describe, expect, it, vi } from 'vitest'; import { createWorkflowRunEvent, getWorkflowRunEvents, @@ -374,6 +375,44 @@ describe('createWorkflowRunEvent result contract', () => { ).rejects.toMatchObject(error); agent.assertNoPendingInterceptors(); }); + + it('does not retry an observer failure that looks like a transport error', async () => { + const agent = mockAgent(); + agent + .get(ORIGIN) + .intercept({ + path: '/api/v4/runs/wrun_1/events/run_started', + method: 'POST', + }) + .reply(200, runStartedResponse(), { + headers: { + 'content-type': V4_FRAME_CONTENT_TYPE, + 'x-wf-event-id': 'evnt_1', + 'x-wf-run-id': 'wrun_1', + 'x-wf-created-at': STARTED_AT.toISOString(), + 'x-wf-max-events': '10000', + }, + }); + + const error = new WorkflowWorldError('observer failed', { + code: 'TRANSPORT', + }); + const onEvent = vi.fn(() => { + throw error; + }); + + await expect( + createWorkflowRunEvent( + 'wrun_1', + { eventType: 'run_started', specVersion: 2 } as AnyEventRequest, + { onEvent }, + { token: 'test-token', dispatcher: agent } + ) + ).rejects.toBe(error); + + expect(onEvent).toHaveBeenCalledOnce(); + agent.assertNoPendingInterceptors(); + }); }); /** POSTs a v4 step_started with `params` and returns the decoded frame meta. */ diff --git a/packages/world-vercel/src/events.ts b/packages/world-vercel/src/events.ts index 49cffb774e..a2adf59827 100644 --- a/packages/world-vercel/src/events.ts +++ b/packages/world-vercel/src/events.ts @@ -31,6 +31,7 @@ * the v3 path. */ +import assert from 'node:assert/strict'; import { HookNotFoundError, WorkflowWorldError } from '@workflow/errors'; import { type AnyEventRequest, @@ -573,12 +574,33 @@ export async function createWorkflowRunEventBatch( }; } +class EventObserverError extends Error { + constructor(readonly error: unknown) { + super('event observer failed'); + } +} + export async function createWorkflowRunEvent( id: string | null, data: T, params?: CreateEventParams, config?: APIConfig ): Promise> { + const onEvent = params?.onEvent; + const requestParams = + onEvent === undefined + ? params + : { + ...params, + onEvent(event: Event) { + try { + onEvent(event); + } catch (error) { + throw new EventObserverError(error); + } + }, + }; + try { // Retry transient transport failures (UND_ERR_REQ_RETRY, ECONNRESET, // socket/headers timeouts, transient 5xx) in-process for event types that @@ -589,7 +611,7 @@ export async function createWorkflowRunEvent( // types (step_started, step_retrying, hook_received) run once. See // ./event-retry for the validated per-event classification. const result = await withEventPostRetry( - () => createWorkflowRunEventInner(id, data, params, config), + () => createWorkflowRunEventInner(id, data, requestParams, config), data.eventType, { // The atomic lazy-resume shape is deduplicated server-side by the @@ -621,6 +643,7 @@ export async function createWorkflowRunEvent( } return result as EventResult; } catch (err) { + if (err instanceof EventObserverError) throw err.error; // 404 on hook_disposed / hook_received → already-disposed hook. if ( isHookEventRequiringExistence(data.eventType) && @@ -756,32 +779,12 @@ async function createWorkflowRunEventInner( 'v4 createEvent: run_started stream is missing run_started' ); } - - let attributes = runCreated.eventData.attributes ?? {}; - let updatedAt = runStarted.createdAt; - for (const event of result.events) { - if (event.eventType === 'attr_set') { - attributes = applyAttributeChanges(attributes, event.eventData.changes); - updatedAt = event.createdAt; - } - } + const run = reconstructRunFromReplayEvents(result.events); + assert(run); return { event: runStarted, - run: { - runId: runCreated.runId, - status: 'running', - deploymentId: runCreated.eventData.deploymentId, - workflowName: runCreated.eventData.workflowName, - specVersion: runCreated.specVersion, - executionContext: runCreated.eventData.executionContext, - input: runCreated.eventData.input, - attributes, - encryptionPublicKey: runCreated.eventData.encryptionPublicKey, - startedAt: runStarted.createdAt, - createdAt: runCreated.createdAt, - updatedAt, - }, + run, events: result.events, cursor: result.cursor, hasMore: result.hasMore, diff --git a/packages/world-vercel/src/http-core.ts b/packages/world-vercel/src/http-core.ts index b8122bb7fd..c0729fbeb1 100644 --- a/packages/world-vercel/src/http-core.ts +++ b/packages/world-vercel/src/http-core.ts @@ -487,6 +487,8 @@ export interface InstrumentedFetchOptions extends HttpClientSpanOptions { * connections stop delivering (see noteEventsTransportOutcome). */ onTransportOutcome?: (error?: unknown) => void; + /** Let a streaming body consumer report success after it finishes. */ + deferTransportSuccess?: boolean; } /** @@ -520,6 +522,7 @@ export async function instrumentedFetch( attributes, durationAttribute, onTransportOutcome, + deferTransportSuccess = false, } = opts; const label = logLabel ?? url; @@ -601,7 +604,9 @@ export async function instrumentedFetch( throw error; } const ms = Date.now() - start; - onTransportOutcome?.(); + if (!deferTransportSuccess || !response.ok) { + onTransportOutcome?.(); + } httpLog(method, label, response, ms); recordClientSpanStatus(span, response.status); From 92aae4c6582fbd7b56bb016eb8589ef947ceb6ad Mon Sep 17 00:00:00 2001 From: Nathan Colosimo <110621881+NathanColosimo@users.noreply.github.com> Date: Thu, 20 Aug 2026 10:20:06 -0700 Subject: [PATCH 8/9] fix(world-vercel): bound materialized v4 requests --- packages/world-vercel/src/events-v4.test.ts | 130 ++++++++++++++++++++ packages/world-vercel/src/events-v4.ts | 13 +- 2 files changed, 136 insertions(+), 7 deletions(-) diff --git a/packages/world-vercel/src/events-v4.test.ts b/packages/world-vercel/src/events-v4.test.ts index a9e966efe1..85b863cd9e 100644 --- a/packages/world-vercel/src/events-v4.test.ts +++ b/packages/world-vercel/src/events-v4.test.ts @@ -804,6 +804,7 @@ describe('v4 transport uses global fetch (observability)', () => { expect(fetchSpy).toHaveBeenCalledTimes(1); const [calledUrl, calledInit] = fetchSpy.mock.calls[0]; expect(String(calledUrl)).toContain('/api/v4/runs/wrun_1/events'); + expect(calledInit?.signal).toBeUndefined(); agent.assertNoPendingInterceptors(); // Cache-busting header must be set so Next.js fetch memoization / Data @@ -812,6 +813,53 @@ describe('v4 transport uses global fetch (observability)', () => { const sentHeaders = new Headers(calledInit?.headers as HeadersInit); expect(sentHeaders.get('x-request-time')).toBeTruthy(); }); + + it('keeps the normal request deadline on a materialized POST', async () => { + const agent = mockAgent(); + agent + .get(ORIGIN) + .intercept({ + path: '/api/v4/runs/wrun_1/events/step_completed', + method: 'POST', + }) + .reply( + 200, + createEventBody( + { + eventType: 'step_completed', + specVersion: 2, + correlationId: 'step_1', + eventData: { result: new Uint8Array() }, + }, + { + step: { + runId: 'wrun_1', + stepId: 'step_1', + stepName: 'step', + status: 'completed', + attempt: 1, + createdAt: CREATED_AT, + updatedAt: CREATED_AT, + }, + } + ) + ); + const fetchSpy = vi.spyOn(globalThis, 'fetch'); + + await createWorkflowRunEventV4( + { + runId: 'wrun_1', + eventType: 'step_completed', + specVersion: 2, + correlationId: 'step_1', + }, + { token: 'test-token', dispatcher: agent } + ); + + const [, calledInit] = fetchSpy.mock.calls[0]; + expect(calledInit?.signal).toBeInstanceOf(AbortSignal); + agent.assertNoPendingInterceptors(); + }); }); describe('createWorkflowRunEventV4 over HTTP', () => { @@ -1132,6 +1180,88 @@ describe('createWorkflowRunEventV4 over HTTP', () => { agent.assertNoPendingInterceptors(); }); + it('retries a truncated continuation that produced no complete event', async () => { + const agent = mockAgent(); + const observed: string[] = []; + const continuationPath = + '/api/v4/runs/wrun_1/events?cursor=eid%3Aevnt_1&remoteRefBehavior=resolve&returnAll=true'; + + agent + .get(ORIGIN) + .intercept({ + path: '/api/v4/runs/wrun_1/events/run_started', + method: 'POST', + headers: { accept: V4_FRAME_CONTENT_TYPE }, + }) + .reply( + 200, + encodeFrame( + { + eventId: 'evnt_1', + runId: 'wrun_1', + eventType: 'run_created', + createdAt: CREATED_AT, + eventData: { + deploymentId: 'dpl_1', + workflowName: 'workflow', + input: null, + }, + }, + new Uint8Array() + ), + { + headers: { + 'content-type': V4_FRAME_CONTENT_TYPE, + 'x-wf-max-events': '10000', + }, + } + ); + + const runStartedFrame = encodeFrame( + { + eventId: 'evnt_2', + runId: 'wrun_1', + eventType: 'run_started', + createdAt: CREATED_AT, + }, + new Uint8Array() + ); + agent + .get(ORIGIN) + .intercept({ path: continuationPath, method: 'GET' }) + .reply(200, runStartedFrame.subarray(0, 4), { + headers: { 'content-type': V4_FRAME_CONTENT_TYPE }, + }); + agent + .get(ORIGIN) + .intercept({ path: continuationPath, method: 'GET' }) + .reply( + 200, + Buffer.concat([ + runStartedFrame, + encodeFrame( + { _end: 1, next: 'eid:evnt_2', hasMore: false }, + new Uint8Array() + ), + ]), + { headers: { 'content-type': V4_FRAME_CONTENT_TYPE } } + ); + + const result = await createWorkflowRunStartedEventV4( + { runId: 'wrun_1', specVersion: 5 }, + { token: 'test-token', dispatcher: agent }, + (event) => observed.push(event.eventId) + ); + + expect(result.events.map((event) => event.eventId)).toEqual([ + 'evnt_1', + 'evnt_2', + ]); + expect(observed).toEqual(['evnt_1', 'evnt_2']); + expect(result.cursor).toBe('eid:evnt_2'); + agent.assertNoPendingInterceptors(); + }); + it('continues a graceful partial run_started stream from its sentinel cursor', async () => { const agent = mockAgent(); diff --git a/packages/world-vercel/src/events-v4.ts b/packages/world-vercel/src/events-v4.ts index cd10aa3eba..78952a662b 100644 --- a/packages/world-vercel/src/events-v4.ts +++ b/packages/world-vercel/src/events-v4.ts @@ -96,9 +96,9 @@ import { isWsEventsTransportEnabled } from './ws-transport-enabled.js'; * stays on HTTP/1.1 because H2 deadlocks the queue's webhook respondWith * mechanism — see http-client.ts. * - * No per-request timeout: a LIST response streams the full event-log page, which - * for a large run can legitimately take a while to drain — a whole-request - * deadline would abort it mid-stream. + * Event streams opt out of the whole-request timeout because a large replay + * page can legitimately take a while to drain. Materialized single and batch + * writes keep the normal request deadline. */ interface V4Response { response: Response; @@ -129,7 +129,7 @@ async function fetchV4( // until the compute instance is recycled — see noteEventsTransportOutcome. onTransportOutcome: reportTransportOutcome, deferTransportSuccess: transportSuccess === 'body', - timeoutMs: null, + timeoutMs: transportSuccess === 'body' ? null : undefined, logLabel: opName, // Read the body as bytes, not text: a CBOR error body (the fence 412 // carries event payloads back) does not survive a UTF-8 decode. @@ -1558,12 +1558,11 @@ export async function getWorkflowRunEventsV4( if ( partialRetries === MAX_PARTIAL_EVENT_STREAM_RETRIES || params.pagination?.limit !== undefined || - !lastEvent || - `eid:${lastEvent.eventId}` === cursor + (!lastEvent && cursor === null) ) { throw result.error; } - cursor = `eid:${lastEvent.eventId}`; + if (lastEvent) cursor = `eid:${lastEvent.eventId}`; continue; } From a0f9f5a24446d1dc4d9ece3af39ea9398f319284 Mon Sep 17 00:00:00 2001 From: Nathan Colosimo <110621881+NathanColosimo@users.noreply.github.com> Date: Thu, 20 Aug 2026 15:35:18 -0700 Subject: [PATCH 9/9] fix(world-vercel): simplify replay stream recovery --- .changeset/resume-partial-replay-streams.md | 3 +- packages/world-vercel/src/event-retry.test.ts | 14 + packages/world-vercel/src/event-retry.ts | 37 +- packages/world-vercel/src/events-v4.test.ts | 139 ++--- packages/world-vercel/src/events-v4.ts | 546 ++++++++---------- packages/world-vercel/src/events.test.ts | 110 ++-- packages/world-vercel/src/events.ts | 47 +- packages/world-vercel/src/frames.test.ts | 33 +- packages/world-vercel/src/frames.ts | 48 +- packages/world-vercel/src/http-core.ts | 2 +- packages/world/src/events.ts | 10 - 11 files changed, 470 insertions(+), 519 deletions(-) diff --git a/.changeset/resume-partial-replay-streams.md b/.changeset/resume-partial-replay-streams.md index 9f5d35e640..1aba3274fe 100644 --- a/.changeset/resume-partial-replay-streams.md +++ b/.changeset/resume-partial-replay-streams.md @@ -1,6 +1,5 @@ --- -"@workflow/world": patch "@workflow/world-vercel": patch --- -Resume interrupted or partial replay event streams after their last validated event and expose decoded events to streaming consumers. +Resume interrupted replay event streams after their last validated event without repeating accepted event writes. diff --git a/packages/world-vercel/src/event-retry.test.ts b/packages/world-vercel/src/event-retry.test.ts index 226888dbc0..ae91530f15 100644 --- a/packages/world-vercel/src/event-retry.test.ts +++ b/packages/world-vercel/src/event-retry.test.ts @@ -9,6 +9,7 @@ import { EventTypeSchema } from '@workflow/world'; import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest'; import { EVENT_RETRY_ELIGIBILITY, + EventPostResponseError, isRetryableEventPostError, MAX_EVENT_POST_RETRIES, THROTTLE_RETRY_BUDGET_MS, @@ -103,6 +104,19 @@ describe('isRetryableEventPostError', () => { ).toBe(true); }); + it('does not repeat a POST to repair a failed response continuation', () => { + const transport = new WorkflowWorldError('continuation failed', { + code: 'TRANSPORT', + }); + expect( + isRetryableEventPostError( + new EventPostResponseError('POST response already accepted', { + cause: transport, + }) + ) + ).toBe(false); + }); + it('retries a TRANSPORT failure from either transport', () => { // The one code that lets this policy serve HTTP and WS alike: `utils.ts` // sets it for a `fetch` that failed transiently, `events-v4.ts` for a WS diff --git a/packages/world-vercel/src/event-retry.ts b/packages/world-vercel/src/event-retry.ts index 3228bd45b8..30dbf6a97e 100644 --- a/packages/world-vercel/src/event-retry.ts +++ b/packages/world-vercel/src/event-retry.ts @@ -76,6 +76,18 @@ import type { z } from 'zod'; * `hook_conflict`, which the SDK never POSTs). */ type WorkflowEventType = z.infer; +/** + * A follow-up read failed after the event POST had already succeeded and + * exposed validated response data. Repeating the mutation cannot repair that + * read and would redownload/re-observe the accepted prefix. + */ +export class EventPostResponseError extends Error { + constructor(message: string, options?: ErrorOptions) { + super(message, options); + this.name = 'EventPostResponseError'; + } +} + export interface EventRetryPolicy { /** Whether a failed POST of this event type may be retried in-process. */ retryable: boolean; @@ -237,14 +249,8 @@ function collectErrorMarkers(err: unknown, depth = 0): string[] { return markers; } -/** - * Whether a failed event POST should be retried as a transient failure. - * Retries transient/ambiguous transport failures and transient 5xx; never - * retries a definitive response (409/410/425/429 and other 4xx). 429 is not - * "transient" in this classification — `withEventPostRetry` gives it its own - * budgeted, Retry-After-honoring policy. - */ -export function isRetryableEventPostError(err: unknown): boolean { +/** Transient request failures shared by idempotent event reads and writes. */ +export function isRetryableEventRequestError(err: unknown): boolean { // Definitive, server-considered outcomes — never retried as *transient*. // (425 is left to the runtime's retry-after handling; 429 has its own // in-process policy in withEventPostRetry, gated by THROTTLE_RETRY_BUDGET_MS @@ -259,9 +265,6 @@ export function isRetryableEventPostError(err: unknown): boolean { } if (WorkflowWorldError.is(err)) { - // Body parsed past the response but the write may have landed — safe to - // retry for eligible events (a landed original re-surfaces as 409). - if (err.code === 'PARSE_ERROR') return true; // A transport failure the world layer already classified as transient: // `utils.ts` sets this for a `fetch` failing with a // `TRANSIENT_TRANSPORT_ERROR_CODES` code, `events-v4.ts` for a WS socket @@ -289,6 +292,18 @@ export function isRetryableEventPostError(err: unknown): boolean { return collectErrorMarkers(err).some((m) => TRANSIENT_CODES.has(m)); } +/** + * Whether a failed event POST should be retried as a transient failure. + * Response parse errors are write-specific: the mutation may have landed, and + * eligible event types converge when it is repeated. A follow-up read failure + * after an accepted POST is explicitly excluded. + */ +export function isRetryableEventPostError(err: unknown): boolean { + if (err instanceof EventPostResponseError) return false; + if (WorkflowWorldError.is(err) && err.code === 'PARSE_ERROR') return true; + return isRetryableEventRequestError(err); +} + const sleep = (ms: number): Promise => new Promise((resolve) => setTimeout(resolve, ms)); diff --git a/packages/world-vercel/src/events-v4.test.ts b/packages/world-vercel/src/events-v4.test.ts index 85b863cd9e..6a2e333df0 100644 --- a/packages/world-vercel/src/events-v4.test.ts +++ b/packages/world-vercel/src/events-v4.test.ts @@ -21,11 +21,12 @@ import { getWorkflowRunEventsV4, throwForErrorResponse, } from './events-v4.js'; -import { encodeFrame, V4_FRAME_CONTENT_TYPE } from './frames.js'; import { - EVENTS_RECYCLE_AFTER_CONSECUTIVE_FAILURES, - getEventsDispatcher, -} from './http-client.js'; + encodeFrame, + InvalidFrameError, + V4_FRAME_CONTENT_TYPE, +} from './frames.js'; +import { getEventsDispatcher } from './http-client.js'; import { WORKFLOW_SERVER_URL_OVERRIDE } from './utils.js'; const CREATED_AT = '2026-06-10T00:00:00.000Z'; @@ -319,7 +320,10 @@ describe('getWorkflowRunEventsV4 over HTTP', () => { agent.assertNoPendingInterceptors(); }); - it('does not resume past a GET observer failure', async () => { + it.each([ + ['an unknown event type', { eventType: 'future_event', eventData: {} }], + ['invalid event metadata', { eventType: 'run_created', eventData: {} }], + ])('rejects %s', async (_description, meta) => { const agent = mockAgent(); agent @@ -331,45 +335,24 @@ describe('getWorkflowRunEventsV4 over HTTP', () => { .reply( 200, Buffer.concat([ - encodeFrame( - { - eventId: 'evnt_1', - runId: 'wrun_1', - eventType: 'run_started', - createdAt: CREATED_AT, - }, - new Uint8Array() - ), - encodeFrame( - { _end: 1, next: 'eid:evnt_1', hasMore: false }, - new Uint8Array() - ), + encodeFrame(meta, new Uint8Array()), + encodeFrame({ _end: 1, hasMore: false }, new Uint8Array()), ]), { headers: { 'content-type': V4_FRAME_CONTENT_TYPE } } ); - const observerError = new WorkflowWorldError('observer failed', { - code: 'TRANSPORT', - }); await expect( getWorkflowRunEventsV4( - { - runId: 'wrun_1', - onEvent: () => { - throw observerError; - }, - }, + { runId: 'wrun_1' }, { token: 'test-token', dispatcher: agent } ) - ).rejects.toBe(observerError); + ).rejects.toThrow(); agent.assertNoPendingInterceptors(); }); - it.each([ - ['an unknown event type', { eventType: 'future_event', eventData: {} }], - ['invalid event metadata', { eventType: 'run_created', eventData: {} }], - ])('rejects %s', async (_description, meta) => { + it('does not retry invalid CBOR frame metadata as a transport failure', async () => { const agent = mockAgent(); + const nonObjectCborFrame = new Uint8Array([0, 0, 0, 1, 0x01, 0, 0, 0, 0]); agent .get(ORIGIN) @@ -377,21 +360,16 @@ describe('getWorkflowRunEventsV4 over HTTP', () => { path: '/api/v4/runs/wrun_1/events?remoteRefBehavior=resolve&returnAll=true', method: 'GET', }) - .reply( - 200, - Buffer.concat([ - encodeFrame(meta, new Uint8Array()), - encodeFrame({ _end: 1, hasMore: false }, new Uint8Array()), - ]), - { headers: { 'content-type': V4_FRAME_CONTENT_TYPE } } - ); + .reply(200, nonObjectCborFrame, { + headers: { 'content-type': V4_FRAME_CONTENT_TYPE }, + }); await expect( getWorkflowRunEventsV4( { runId: 'wrun_1' }, { token: 'test-token', dispatcher: agent } ) - ).rejects.toThrow(); + ).rejects.toBeInstanceOf(InvalidFrameError); agent.assertNoPendingInterceptors(); }); @@ -614,7 +592,7 @@ describe('getWorkflowRunEventsV4 over HTTP', () => { agent.assertNoPendingInterceptors(); }); - it('rejects an unexpected clean pagination response', async () => { + it('preserves a clean event-ceiling response', async () => { const agent = mockAgent(); agent .get(ORIGIN) @@ -631,12 +609,16 @@ describe('getWorkflowRunEventsV4 over HTTP', () => { { headers: { 'content-type': V4_FRAME_CONTENT_TYPE } } ); - await expect( - getWorkflowRunEventsV4( - { runId: 'wrun_1' }, - { token: 'test-token', dispatcher: agent } - ) - ).rejects.toThrow(/returnAll response was unexpectedly paginated/); + const result = await getWorkflowRunEventsV4( + { runId: 'wrun_1' }, + { token: 'test-token', dispatcher: agent } + ); + + expect(result).toEqual({ + data: [], + cursor: 'eid:evnt_1', + hasMore: true, + }); agent.assertNoPendingInterceptors(); }); }); @@ -1039,7 +1021,6 @@ describe('createWorkflowRunEventV4 over HTTP', () => { it('continues a truncated run_started stream after its last event', async () => { const agent = mockAgent(); - const observed: string[] = []; agent .get(ORIGIN) @@ -1099,8 +1080,7 @@ describe('createWorkflowRunEventV4 over HTTP', () => { const result = await createWorkflowRunStartedEventV4( { runId: 'wrun_1', specVersion: 5 }, - { token: 'test-token', dispatcher: agent }, - (event) => observed.push(event.eventId) + { token: 'test-token', dispatcher: agent } ); expect(result.events.map((event) => event.eventId)).toEqual([ @@ -1108,7 +1088,6 @@ describe('createWorkflowRunEventV4 over HTTP', () => { 'evnt_2', ]); expect(result.hasMore).toBe(false); - expect(observed).toEqual(['evnt_1', 'evnt_2']); agent.assertNoPendingInterceptors(); }); @@ -1182,7 +1161,6 @@ describe('createWorkflowRunEventV4 over HTTP', () => { it('retries a truncated continuation that produced no complete event', async () => { const agent = mockAgent(); - const observed: string[] = []; const continuationPath = '/api/v4/runs/wrun_1/events?cursor=eid%3Aevnt_1&remoteRefBehavior=resolve&returnAll=true'; @@ -1249,20 +1227,18 @@ describe('createWorkflowRunEventV4 over HTTP', () => { const result = await createWorkflowRunStartedEventV4( { runId: 'wrun_1', specVersion: 5 }, - { token: 'test-token', dispatcher: agent }, - (event) => observed.push(event.eventId) + { token: 'test-token', dispatcher: agent } ); expect(result.events.map((event) => event.eventId)).toEqual([ 'evnt_1', 'evnt_2', ]); - expect(observed).toEqual(['evnt_1', 'evnt_2']); expect(result.cursor).toBe('eid:evnt_2'); agent.assertNoPendingInterceptors(); }); - it('continues a graceful partial run_started stream from its sentinel cursor', async () => { + it('preserves a clean event-ceiling run_started response', async () => { const agent = mockAgent(); agent @@ -1301,43 +1277,14 @@ describe('createWorkflowRunEventV4 over HTTP', () => { }, } ); - agent - .get(ORIGIN) - .intercept({ - path: '/api/v4/runs/wrun_1/events?cursor=eid%3Aevnt_1&remoteRefBehavior=resolve&returnAll=true', - method: 'GET', - }) - .reply( - 200, - Buffer.concat([ - encodeFrame( - { - eventId: 'evnt_2', - runId: 'wrun_1', - eventType: 'run_started', - createdAt: CREATED_AT, - }, - new Uint8Array() - ), - encodeFrame( - { _end: 1, next: 'eid:evnt_2', hasMore: false }, - new Uint8Array() - ), - ]), - { headers: { 'content-type': V4_FRAME_CONTENT_TYPE } } - ); - const result = await createWorkflowRunStartedEventV4( { runId: 'wrun_1', specVersion: 5 }, { token: 'test-token', dispatcher: agent } ); - expect(result.events.map((event) => event.eventId)).toEqual([ - 'evnt_1', - 'evnt_2', - ]); - expect(result.cursor).toBe('eid:evnt_2'); - expect(result.hasMore).toBe(false); + expect(result.events.map((event) => event.eventId)).toEqual(['evnt_1']); + expect(result.cursor).toBe('eid:evnt_1'); + expect(result.hasMore).toBe(true); agent.assertNoPendingInterceptors(); }); @@ -1993,16 +1940,12 @@ describe('v4 transport reports failures to the events recycler', () => { // which is what the recycler owns. const before = getEventsDispatcher({ token: 'test-token' }); - for (let i = 0; i < EVENTS_RECYCLE_AFTER_CONSECUTIVE_FAILURES; i++) { - await expect( - getWorkflowRunEventsV4({ runId: 'wrun_1' }, { token: 'test-token' }) - ).rejects.toThrow(); - // Still the same pool until the threshold is reached. - if (i < EVENTS_RECYCLE_AFTER_CONSECUTIVE_FAILURES - 1) { - expect(getEventsDispatcher({ token: 'test-token' })).toBe(before); - } - } + await expect( + getWorkflowRunEventsV4({ runId: 'wrun_1' }, { token: 'test-token' }) + ).rejects.toThrow(); + // The bounded GET retry loop produces enough consecutive body failures to + // retire the wedged pool within this one logical read. expect(getEventsDispatcher({ token: 'test-token' })).not.toBe(before); }); }); diff --git a/packages/world-vercel/src/events-v4.ts b/packages/world-vercel/src/events-v4.ts index 78952a662b..2fc003b5d5 100644 --- a/packages/world-vercel/src/events-v4.ts +++ b/packages/world-vercel/src/events-v4.ts @@ -40,14 +40,20 @@ import { } from '@workflow/world'; import { decode } from 'cbor-x'; import { z } from 'zod'; +import { + EventPostResponseError, + isRetryableEventRequestError, +} from './event-retry.js'; import { type DecodedFrame, decodeFrames, encodeFrame, + IncompleteFrameError, V4_FRAME_CONTENT_TYPE, } from './frames.js'; import { getEventsDispatcher, + isRecyclableTransportError, noteEventsTransportOutcome, } from './http-client.js'; import { @@ -100,36 +106,35 @@ import { isWsEventsTransportEnabled } from './ws-transport-enabled.js'; * page can legitimately take a while to drain. Materialized single and batch * writes keep the normal request deadline. */ -interface V4Response { - response: Response; - reportTransportOutcome(error?: unknown): void; +interface V4Request { + url: string; + init: { method: string; headers: Headers; body?: Uint8Array }; + config?: APIConfig; + opName: string; + attributes?: Record; } -async function fetchV4( - url: string, - init: { method: string; headers: Headers; body?: Uint8Array }, - config: APIConfig | undefined, - opName: string, - transportSuccess: 'headers' | 'body', - attributes?: Record -): Promise { - const dispatcher = getEventsDispatcher(config); - const reportTransportOutcome = (error?: unknown) => - noteEventsTransportOutcome(dispatcher, error); - const response = await instrumentedFetch({ - method: init.method, +function instrumentedV4Fetch( + request: V4Request, + options?: { + dispatcher?: ReturnType; + deferTransportSuccess?: boolean; + onTransportOutcome?: (error?: unknown) => void; + } +): Promise { + const { url, init, config, opName, attributes } = request; + return instrumentedFetch({ + ...init, url, - headers: init.headers, - body: init.body, - dispatcher, + dispatcher: options?.dispatcher ?? getEventsDispatcher(config), attributes, // Repeated transport failures retire the shared events pool and the next // request builds a fresh one. undici keeps a black-holed HTTP/2 session in // service indefinitely, so without this every request routed onto it fails // until the compute instance is recycled — see noteEventsTransportOutcome. - onTransportOutcome: reportTransportOutcome, - deferTransportSuccess: transportSuccess === 'body', - timeoutMs: transportSuccess === 'body' ? null : undefined, + onTransportOutcome: options?.onTransportOutcome, + deferTransportSuccess: options?.deferTransportSuccess, + timeoutMs: options?.deferTransportSuccess ? null : undefined, logLabel: opName, // Read the body as bytes, not text: a CBOR error body (the fence 412 // carries event payloads back) does not survive a UTF-8 decode. @@ -142,7 +147,53 @@ async function fetchV4( url ), }); - return { response, reportTransportOutcome }; +} + +/** Materialized responses are transport-complete once their headers arrive. */ +function fetchV4(request: V4Request): Promise { + const dispatcher = getEventsDispatcher(request.config); + return instrumentedV4Fetch(request, { + dispatcher, + onTransportOutcome: (error) => + noteEventsTransportOutcome(dispatcher, error), + }); +} + +/** + * Keep transport accounting scoped to the full response body. Protocol errors + * prove that the origin answered and therefore count as transport success; + * incomplete bodies preserve/report the transport failure instead. + */ +async function withV4ResponseBody( + request: V4Request, + consume: (response: Response) => Promise +): Promise { + const dispatcher = getEventsDispatcher(request.config); + let outcomeReported = false; + const report = (error?: unknown) => { + outcomeReported = true; + noteEventsTransportOutcome(dispatcher, error); + }; + + try { + const response = await instrumentedV4Fetch(request, { + dispatcher, + deferTransportSuccess: true, + onTransportOutcome: report, + }); + const result = await consume(response); + report(); + return result; + } catch (error) { + if (!outcomeReported) { + const incomplete = + error instanceof IncompleteFrameError || + error instanceof PartialEventStreamError || + isRecyclableTransportError(error); + report(incomplete ? error : undefined); + } + throw error; + } } const EVENT_ID_HEADER = 'x-wf-event-id'; @@ -695,18 +746,18 @@ export function throwForErrorResponse( * The frame meta's `eventType` remains authoritative — the backend * cross-checks the two and logs (but does not reject) a mismatch. */ -async function postWorkflowRunEventV4( +async function workflowRunEventV4Request( input: CreateEventV4InputBase & { eventType: EventType; skipPreload?: true; }, - responseType: 'materialized' | 'event-stream', + eventStream: boolean, config?: APIConfig -) { +): Promise { const { baseUrl, headers: baseHeaders } = await getHttpConfig(config); const headers = new Headers(baseHeaders); headers.set('Content-Type', 'application/octet-stream'); - if (responseType === 'event-stream') { + if (eventStream) { headers.set('Accept', V4_FRAME_CONTENT_TYPE); } @@ -716,13 +767,12 @@ async function postWorkflowRunEventV4( ); const url = eventsV4Url(baseUrl, input.runId, input.eventType); - return fetchV4( + return { url, - { method: 'POST', headers, body: frame }, + init: { method: 'POST', headers, body: frame }, config, - 'createEvent', - responseType === 'event-stream' ? 'body' : 'headers', - { + opName: 'createEvent', + attributes: { ...WorkflowEventsTransport('http'), ...WorkflowEventType(input.eventType), ...WorkflowClientVersion(`@workflow/world-vercel/${version}`), @@ -730,7 +780,31 @@ async function postWorkflowRunEventV4( ...(input.optimizations !== undefined ? StepLatencyOptimizations(input.optimizations) : {}), - } + }, + }; +} + +async function postWorkflowRunEventV4( + input: CreateEventV4InputBase & { + eventType: EventType; + skipPreload?: true; + }, + config?: APIConfig +): Promise { + return fetchV4(await workflowRunEventV4Request(input, false, config)); +} + +async function withWorkflowRunEventResponseBody( + input: CreateEventV4InputBase & { + eventType: EventType; + skipPreload?: true; + }, + config: APIConfig | undefined, + consume: (response: Response) => Promise +): Promise { + return withV4ResponseBody( + await workflowRunEventV4Request(input, true, config), + consume ); } @@ -751,11 +825,7 @@ export async function createWorkflowRunEventV4( if (reply) return decodeCreateEventResponse(reply, input.eventType); } - const { response } = await postWorkflowRunEventV4( - input, - 'materialized', - config - ); + const response = await postWorkflowRunEventV4(input, config); const contentType = response.headers.get('content-type'); if (contentType?.startsWith(V4_FRAME_CONTENT_TYPE)) { @@ -795,22 +865,15 @@ async function decodeCreateEventResponse( export async function createWorkflowRunStartedEventV4( input: CreateEventV4InputBase, - config?: APIConfig, - onEvent?: (event: Event) => void + config?: APIConfig ) { - const stream = await postWorkflowRunEventV4( + const { responseHeaders, ...replay } = await postReplayLogEvent( { ...input, eventType: 'run_started' }, - 'event-stream', - config - ); - const replay = await consumeReplayLogResponse( - stream, - { runId: input.runId, onEvent }, config ); assert(replay.cursor, 'v4 createEvent: event stream missing cursor'); const maxEvents = MaxEventsHeaderSchema.safeParse( - stream.response.headers.get(MAX_EVENTS_HEADER) + responseHeaders.get(MAX_EVENTS_HEADER) ); if (!maxEvents.success) { throw new WorkflowWorldError('v4 createEvent: invalid max-events header', { @@ -907,17 +970,16 @@ export async function createWorkflowRunEventsBatchV4( // span carries only wire-level facts. workflow.event.type is deliberately // absent — it names a single event write, and tagging a batch with its // first event's type misclassifies the traffic. - const { response } = await fetchV4( + const response = await fetchV4({ url, - { method: 'POST', headers, body }, + init: { method: 'POST', headers, body }, config, - 'createEventBatch', - 'headers', - { + opName: 'createEventBatch', + attributes: { ...WorkflowEventsTransport('http'), 'workflow.batch.bytes': body.byteLength, - } - ); + }, + }); const bodyBytes = new Uint8Array(await response.arrayBuffer()); const decoded = @@ -1204,84 +1266,41 @@ async function postEventFrameOverWs( ); } -export interface ReplayLogResult { +interface ReplayLog { events: Event[]; cursor: string | null; hasMore: boolean; } -/** - * Result of a `hook_received` POST that opted into the replay-log preload, - * discriminated on `kind` (keyed on the response content type). - */ -export type HookReceivedPreloadV4Result = - /** The server streamed the replay log back as v4 frames. */ - | (ReplayLogResult & { - kind: 'stream'; - /** - * The canonical event this write created or converged on (the resume - * claim winner's — ours or the producer's), named by the - * event-id response header. Undefined when the server did not send it. - */ - canonicalEventId: string | undefined; - /** Per-run event ceiling from the response header, when present. */ - maxEvents: number | undefined; - }) - /** - * The server answered with the normal materialized CBOR body instead — - * an older server, or one that declined the optimization. The - * hook_received write itself has still succeeded; callers must not - * re-post it. - */ - | { - kind: 'materialized'; - result: EventResult<'hook_received'> & { event: Event }; - }; - /** * POST /api/v4/runs/:runId/events/hook_received with the v4-frame `Accept`, - * consuming either response mode. + * consuming the replay-log response. * - * A server that supports the lazy-hook replay stream answers the consumer's - * idempotent re-ensure with the run's complete replay log as v4 frames — - * the same event-frame sequence LIST uses, ending with the `_end` sentinel. - * A truncated stream resumes with a GET after its last validated event. If it - * fails before producing any event, the outer POST retry remains safe because - * the server deduplicates `(runId, resumeId)` and returns the canonical event. + * The consumer's idempotent re-ensure always answers with the run's replay log + * as v4 frames. A truncated stream resumes with a GET after its last validated + * event. If it fails before producing any event, the outer POST retry remains + * safe because the server deduplicates `(runId, resumeId)` and returns the + * canonical event. */ export async function createHookReceivedPreloadEventV4( input: CreateEventV4InputBase, - config?: APIConfig, - onEvent?: (event: Event) => void -): Promise { - const stream = await postWorkflowRunEventV4( - { ...input, eventType: 'hook_received' }, - 'event-stream', - config - ); - - const { response } = stream; - const contentType = response.headers.get('content-type'); - if (!contentType?.startsWith(V4_FRAME_CONTENT_TYPE)) { - stream.reportTransportOutcome(); - return { - kind: 'materialized', - result: await decodeCreateEventResponse(response, 'hook_received'), - }; + config?: APIConfig +): Promise< + ReplayLog & { + canonicalEventId: string | undefined; + maxEvents: number | undefined; } - - const replay = await consumeReplayLogResponse( - stream, - { runId: input.runId, onEvent }, +> { + const { responseHeaders, ...replay } = await postReplayLogEvent( + { ...input, eventType: 'hook_received' }, config ); const maxEvents = MaxEventsHeaderSchema.safeParse( - response.headers.get(MAX_EVENTS_HEADER) + responseHeaders.get(MAX_EVENTS_HEADER) ); return { - kind: 'stream', ...replay, - canonicalEventId: response.headers.get(EVENT_ID_HEADER) ?? undefined, + canonicalEventId: responseHeaders.get(EVENT_ID_HEADER) ?? undefined, maxEvents: maxEvents.success ? maxEvents.data : undefined, }; } @@ -1313,43 +1332,46 @@ export async function getEventV4( const url = `${baseUrl}/v4/runs/${encodeURIComponent(runId)}/events/${encodeURIComponent(eventId)}` + `?remoteRefBehavior=${remoteRefBehavior}`; - const stream = await fetchV4( - url, - { method: 'GET', headers }, - config, - 'getEvent', - 'body' + return withV4ResponseBody( + { + url, + init: { method: 'GET', headers }, + config, + opName: 'getEvent', + }, + async (response) => { + // GET emits one frame without a sentinel. + for await (const frame of decodeFrames( + eventFrameChunks(response, 'getEvent') + )) { + return decodeEventFrame(frame); + } + throw new IncompleteFrameError( + `v4 getEvent: empty frame stream for ${eventId}` + ); + } ); - const { response } = stream; +} + +function eventFrameChunks( + response: Response, + opName: string +): AsyncIterable { const contentType = response.headers.get('content-type'); if (!contentType?.startsWith(V4_FRAME_CONTENT_TYPE)) { - stream.reportTransportOutcome(); throw new Error( - `v4 getEvent: expected ${V4_FRAME_CONTENT_TYPE}, got ${contentType ?? '(none)'}` + `v4 ${opName}: expected ${V4_FRAME_CONTENT_TYPE}, got ${contentType ?? '(none)'}` ); } + if (!response.body) { + throw new IncompleteFrameError(`v4 ${opName}: response body is missing`); + } // fetch's `Response.body` is a web ReadableStream, which is async-iterable - // on Node (readableStream async iteration, since v16.5.0) — feed it straight - // to decodeFrames. The cast is only because TS's lib `ReadableStream` type - // omits the async iterator. Do NOT round-trip through `node:stream` - // Readable.toWeb: a dynamic `import('node:stream')` resolves to an empty - // module namespace in Next.js webpack server bundles and crashes. - const chunks = response.body as unknown as AsyncIterable; - - // GET emits a single frame (no sentinel); decodeFrames returns at EOF - // after yielding it. - try { - for await (const frame of decodeFrames(chunks)) { - stream.reportTransportOutcome(); - return decodeEventFrame(frame); - } - stream.reportTransportOutcome(); - } catch (error) { - stream.reportTransportOutcome(error); - throw error; - } - throw new Error(`v4 getEvent: empty frame stream for ${eventId}`); + // on Node. The cast is only because TS's lib `ReadableStream` type omits the + // async iterator. A node:stream conversion breaks in Next.js webpack server + // bundles, where that dynamic import resolves to an empty namespace. + return response.body as unknown as AsyncIterable; } class PartialEventStreamError extends WorkflowWorldError { @@ -1360,131 +1382,87 @@ class PartialEventStreamError extends WorkflowWorldError { const MAX_PARTIAL_EVENT_STREAM_RETRIES = 3; -type EventFrameStreamResult = - | { - kind: 'complete'; - cursor: string | null; - hasMore: boolean; - } - | { - kind: 'partial'; - error: PartialEventStreamError; - }; - async function consumeEventFrameStream( - stream: V4Response, + response: Response, opName: string, - events: Event[], - onEvent?: (event: Event) => void -): Promise { - const { response } = stream; - const contentType = response.headers.get('content-type'); - if (!contentType?.startsWith(V4_FRAME_CONTENT_TYPE)) { - stream.reportTransportOutcome(); - throw new Error( - `v4 ${opName}: expected ${V4_FRAME_CONTENT_TYPE}, got ${contentType ?? '(none)'}` - ); - } - - const chunks = response.body as unknown as AsyncIterable; - const frames = decodeFrames(chunks); - + events: Event[] +): Promise<{ cursor: string | null; hasMore: boolean }> { try { - while (true) { - let next: IteratorResult; - try { - next = await frames.next(); - } catch (cause) { - const error = new PartialEventStreamError( - `v4 ${opName}: event frame stream failed after ${events.length} events`, - cause - ); - stream.reportTransportOutcome(error); - return { kind: 'partial', error }; + for await (const frame of decodeFrames( + eventFrameChunks(response, opName) + )) { + if (frame.meta._end === 1) { + const end = EventStreamEndSchema.parse(frame.meta); + return { + cursor: end.next ?? null, + hasMore: end.hasMore, + }; } - if (next.done) break; - - try { - const frame = next.value; - if (frame.meta._end === 1) { - const end = EventStreamEndSchema.parse(frame.meta); - stream.reportTransportOutcome(); - return { - kind: 'complete', - cursor: end.next ?? null, - hasMore: end.hasMore, - }; - } - if (Object.keys(frame.meta).some((key) => key.startsWith('_'))) { - throw new Error(`v4 ${opName}: unexpected control frame`); - } - const event = decodeEventFrame(frame); - events.push(event); - onEvent?.(event); - } catch (error) { - stream.reportTransportOutcome(); - throw error; + if (Object.keys(frame.meta).some((key) => key.startsWith('_'))) { + throw new Error(`v4 ${opName}: unexpected control frame`); } + events.push(decodeEventFrame(frame)); } - } finally { - void frames.return(undefined); + } catch (cause) { + if (!(cause instanceof IncompleteFrameError)) throw cause; + throw new PartialEventStreamError( + `v4 ${opName}: event frame stream failed after ${events.length} events`, + cause + ); } - const error = new PartialEventStreamError( + throw new PartialEventStreamError( `v4 ${opName}: frame stream ended without the end-of-stream sentinel ` + `(${events.length} events read) — truncated response?` ); - stream.reportTransportOutcome(error); - return { kind: 'partial', error }; } -/** - * Finish a replay-log response, reusing every validated prefix. A graceful - * `hasMore` sentinel and an interrupted body now converge on the same GET - * continuation instead of making run_started download its accepted prefix - * again. - */ -async function consumeReplayLogResponse( - stream: V4Response, - params: Pick, +/** POST a replay log and continue only an actually incomplete response body. */ +async function postReplayLogEvent( + input: CreateEventV4InputBase & { + eventType: 'run_started' | 'hook_received'; + }, config?: APIConfig -): Promise { +): Promise { const events: Event[] = []; - let page: { cursor: string | null; hasMore: boolean }; - const consumed = await consumeEventFrameStream( - stream, - 'createEvent', - events, - params.onEvent - ); - if (consumed.kind === 'partial') { + let responseHeaders: Headers | undefined; + try { + const page = await withWorkflowRunEventResponseBody( + input, + config, + (response) => { + responseHeaders = response.headers; + return consumeEventFrameStream(response, 'createEvent', events); + } + ); + assert(responseHeaders); + return { events, ...page, responseHeaders }; + } catch (error) { + if (!(error instanceof PartialEventStreamError)) throw error; const lastEvent = events.at(-1); - if (!lastEvent) throw consumed.error; - page = { cursor: `eid:${lastEvent.eventId}`, hasMore: true }; - } else { - page = consumed; - } + if (!lastEvent) throw error; + assert(responseHeaders); + const continuationCursor = `eid:${lastEvent.eventId}`; - if (!page.hasMore) return { events, ...page }; - assert(page.cursor, 'v4 createEvent: partial event stream missing cursor'); - const suffix = await getWorkflowRunEventsV4( - { - runId: params.runId, - pagination: { cursor: page.cursor }, - onEvent: params.onEvent, - }, - config - ); - events.push(...suffix.data); - assert( - suffix.data.length === 0 || suffix.cursor, - 'v4 createEvent: non-empty continuation missing cursor' - ); - return { - events, - cursor: suffix.cursor ?? page.cursor, - hasMore: suffix.hasMore, - }; + try { + const suffix = await getWorkflowRunEventsV4( + { runId: input.runId, pagination: { cursor: continuationCursor } }, + config + ); + events.push(...suffix.data); + return { + events, + cursor: suffix.cursor ?? continuationCursor, + hasMore: suffix.hasMore, + responseHeaders, + }; + } catch (cause) { + throw new EventPostResponseError( + `v4 createEvent: replay continuation failed for run ${input.runId}`, + { cause } + ); + } + } } /** @@ -1536,43 +1514,32 @@ export async function getWorkflowRunEventsV4( const events: Event[] = []; let cursor = params.pagination?.cursor ?? null; - for (let partialRetries = 0; ; partialRetries++) { + for (let retries = 0; ; retries++) { const url = `${baseUrl}/v4/runs/${encodeURIComponent(params.runId)}/events` + paginationToQuery(params, cursor); - const stream = await fetchV4( - url, - { method: 'GET', headers }, - config, - 'listEvents', - 'body' - ); - const result = await consumeEventFrameStream( - stream, - 'listEvents', - events, - params.onEvent - ); - if (result.kind === 'partial') { + try { + const result = await withV4ResponseBody( + { + url, + init: { method: 'GET', headers }, + config, + opName: 'listEvents', + }, + (response) => consumeEventFrameStream(response, 'listEvents', events) + ); + return { data: events, cursor: result.cursor, hasMore: result.hasMore }; + } catch (error) { const lastEvent = events.at(-1); if ( - partialRetries === MAX_PARTIAL_EVENT_STREAM_RETRIES || + retries === MAX_PARTIAL_EVENT_STREAM_RETRIES || params.pagination?.limit !== undefined || - (!lastEvent && cursor === null) + !isRetryableEventRequestError(error) ) { - throw result.error; + throw error; } if (lastEvent) cursor = `eid:${lastEvent.eventId}`; - continue; - } - - if (params.pagination?.limit === undefined && result.hasMore) { - throw new WorkflowWorldError( - `v4 listEvents: returnAll response was unexpectedly paginated for run ${params.runId}`, - { code: 'SCHEMA_VALIDATION' } - ); } - return { data: events, cursor: result.cursor, hasMore: result.hasMore }; } } @@ -1601,18 +1568,15 @@ export async function getEventsByCorrelationIdV4( appendListParams(sp, params, params.pagination?.cursor ?? null); const url = `${baseUrl}/v4/events?${sp.toString()}`; const events: Event[] = []; - const stream = await fetchV4( - url, - { method: 'GET', headers }, - config, - 'listEventsByCorrelationId', - 'body' - ); - const result = await consumeEventFrameStream( - stream, - 'listEventsByCorrelationId', - events + const result = await withV4ResponseBody( + { + url, + init: { method: 'GET', headers }, + config, + opName: 'listEventsByCorrelationId', + }, + (response) => + consumeEventFrameStream(response, 'listEventsByCorrelationId', events) ); - if (result.kind === 'partial') throw result.error; return { data: events, cursor: result.cursor, hasMore: result.hasMore }; } diff --git a/packages/world-vercel/src/events.test.ts b/packages/world-vercel/src/events.test.ts index 87ba906319..c396ad9724 100644 --- a/packages/world-vercel/src/events.test.ts +++ b/packages/world-vercel/src/events.test.ts @@ -1,11 +1,11 @@ import { Buffer } from 'node:buffer'; import { gzipSync } from 'node:zlib'; -import { WorkflowWorldError } from '@workflow/errors'; import type { AnyEventRequest, CreateEventParams } from '@workflow/world'; import { decode, encode } from 'cbor-x'; import { ulid } from 'ulid'; import { MockAgent } from 'undici'; -import { describe, expect, it, vi } from 'vitest'; +import { describe, expect, it } from 'vitest'; +import { EventPostResponseError } from './event-retry.js'; import { createWorkflowRunEvent, getWorkflowRunEvents, @@ -376,7 +376,7 @@ describe('createWorkflowRunEvent result contract', () => { agent.assertNoPendingInterceptors(); }); - it('does not retry an observer failure that looks like a transport error', async () => { + it('does not repeat a run_started POST when its continuation exhausts retries', async () => { const agent = mockAgent(); agent .get(ORIGIN) @@ -384,33 +384,49 @@ describe('createWorkflowRunEvent result contract', () => { path: '/api/v4/runs/wrun_1/events/run_started', method: 'POST', }) - .reply(200, runStartedResponse(), { - headers: { - 'content-type': V4_FRAME_CONTENT_TYPE, - 'x-wf-event-id': 'evnt_1', - 'x-wf-run-id': 'wrun_1', - 'x-wf-created-at': STARTED_AT.toISOString(), - 'x-wf-max-events': '10000', - }, - }); + .reply( + 200, + encodeFrame( + { + eventId: 'evnt_0', + runId: 'wrun_1', + eventType: 'run_created', + createdAt: new Date('2026-06-09T23:59:59.000Z'), + specVersion: 2, + eventData: { + deploymentId: 'dpl_1', + workflowName: 'workflow', + }, + }, + new Uint8Array() + ), + { + headers: { + 'content-type': V4_FRAME_CONTENT_TYPE, + 'x-wf-max-events': '10000', + }, + } + ); - const error = new WorkflowWorldError('observer failed', { - code: 'TRANSPORT', - }); - const onEvent = vi.fn(() => { - throw error; - }); + const continuationPath = + '/api/v4/runs/wrun_1/events?cursor=eid%3Aevnt_0&remoteRefBehavior=resolve&returnAll=true'; + for (let attempt = 0; attempt < 4; attempt++) { + agent + .get(ORIGIN) + .intercept({ path: continuationPath, method: 'GET' }) + .reply(200, new Uint8Array(), { + headers: { 'content-type': V4_FRAME_CONTENT_TYPE }, + }); + } await expect( createWorkflowRunEvent( 'wrun_1', { eventType: 'run_started', specVersion: 2 } as AnyEventRequest, - { onEvent }, + undefined, { token: 'test-token', dispatcher: agent } ) - ).rejects.toBe(error); - - expect(onEvent).toHaveBeenCalledOnce(); + ).rejects.toBeInstanceOf(EventPostResponseError); agent.assertNoPendingInterceptors(); }); }); @@ -1703,9 +1719,8 @@ describe('createWorkflowRunEvent hook_received replay preload', () => { ); // The idempotency key + digest rode the frame meta. The request keeps - // hook_received's lazy default: a supporting server owns frame-body - // resolution regardless, and an older server then answers the CBOR - // fallback without resolving a payload the runtime would discard. + // hook_received's lazy default; the preload contract makes the server own + // resolution of the replay-ready frame bodies. expect(capturedMeta?.resumeId).toBe(RESUME_ID); expect(capturedMeta?.resumePayloadDigest).toBe(DIGEST); expect(capturedMeta?.remoteRefBehavior).toBe('lazy'); @@ -1844,7 +1859,7 @@ describe('createWorkflowRunEvent hook_received replay preload', () => { agent.assertNoPendingInterceptors(); }); - it('keeps the CBOR result when the server does not stream (older server)', async () => { + it('requires the framed preload response promised by workflow-server', async () => { const agent = mockAgent(); agent .get(ORIGIN) @@ -1853,41 +1868,18 @@ describe('createWorkflowRunEvent hook_received replay preload', () => { method: 'POST', headers: { accept: V4_FRAME_CONTENT_TYPE }, }) - .reply( - 200, - encode({ - event: { - eventId: 'evnt_4', - runId: 'wrun_1', - eventType: 'hook_received', - correlationId: 'hook_1', - createdAt: new Date('2026-06-10T00:00:03.000Z'), - specVersion: 2, - eventData: { token: 'tok-preload' }, - }, - }), - { - headers: { - 'content-type': 'application/cbor', - 'x-wf-event-id': 'evnt_4', - 'x-wf-run-id': 'wrun_1', - 'x-wf-created-at': '2026-06-10T00:00:03.000Z', - }, - } - ); + .reply(200, encode({ event: { eventType: 'hook_received' } }), { + headers: { 'content-type': 'application/cbor' }, + }); - const result = await createWorkflowRunEvent( - 'wrun_1', - hookReceivedRequest(), - preloadParams, - { token: 'test-token', dispatcher: agent } + await expect( + createWorkflowRunEvent('wrun_1', hookReceivedRequest(), preloadParams, { + token: 'test-token', + dispatcher: agent, + }) + ).rejects.toThrow( + `v4 createEvent: expected ${V4_FRAME_CONTENT_TYPE}, got application/cbor` ); - - // A successful write with no replay preload — the runtime falls back to - // the run_started setup without posting the hook again. - expect(result.event?.eventType).toBe('hook_received'); - expect(result.events).toBeUndefined(); - expect(result.run).toBeUndefined(); agent.assertNoPendingInterceptors(); }); diff --git a/packages/world-vercel/src/events.ts b/packages/world-vercel/src/events.ts index a2adf59827..d2903ff116 100644 --- a/packages/world-vercel/src/events.ts +++ b/packages/world-vercel/src/events.ts @@ -574,33 +574,12 @@ export async function createWorkflowRunEventBatch( }; } -class EventObserverError extends Error { - constructor(readonly error: unknown) { - super('event observer failed'); - } -} - export async function createWorkflowRunEvent( id: string | null, data: T, params?: CreateEventParams, config?: APIConfig ): Promise> { - const onEvent = params?.onEvent; - const requestParams = - onEvent === undefined - ? params - : { - ...params, - onEvent(event: Event) { - try { - onEvent(event); - } catch (error) { - throw new EventObserverError(error); - } - }, - }; - try { // Retry transient transport failures (UND_ERR_REQ_RETRY, ECONNRESET, // socket/headers timeouts, transient 5xx) in-process for event types that @@ -611,7 +590,7 @@ export async function createWorkflowRunEvent( // types (step_started, step_retrying, hook_received) run once. See // ./event-retry for the validated per-event classification. const result = await withEventPostRetry( - () => createWorkflowRunEventInner(id, data, requestParams, config), + () => createWorkflowRunEventInner(id, data, params, config), data.eventType, { // The atomic lazy-resume shape is deduplicated server-side by the @@ -643,7 +622,6 @@ export async function createWorkflowRunEvent( } return result as EventResult; } catch (err) { - if (err instanceof EventObserverError) throw err.error; // 404 on hook_disposed / hook_received → already-disposed hook. if ( isHookEventRequiringExistence(data.eventType) && @@ -758,11 +736,7 @@ async function createWorkflowRunEventInner( }; if (data.eventType === 'run_started' && !params?.skipPreload) { - const result = await createWorkflowRunStartedEventV4( - input, - config, - params?.onEvent - ); + const result = await createWorkflowRunStartedEventV4(input, config); const runCreated = result.events.find( (event) => event.eventType === 'run_created' ); @@ -800,24 +774,15 @@ async function createWorkflowRunEventInner( ) { // Lazy hook resume: the queue consumer's idempotent re-ensure doubles // as the invocation's setup request. A supporting server streams the - // complete replay log back in this response with resolved frame bodies - // — the SERVER owns that resolution (the preload contract requires + // complete replay log back in this response with resolved frame bodies. + // The SERVER owns that resolution (the preload contract requires // replay-ready bytes; v4 has no /refs endpoint to hydrate a lazy // descriptor during replay), so the request keeps hook_received's lazy - // default. Against an older server this makes the CBOR fallback - // lightweight: it answers the mutation without resolving and echoing - // an S3-backed hook payload the runtime would discard anyway. + // default. const outcome = await createHookReceivedPreloadEventV4( { ...input, remoteRefBehavior: 'lazy' }, - config, - params.onEvent + config ); - if (outcome.kind === 'materialized') { - // Older server (or optimization declined): the write still succeeded - // and this is its normal materialized result. The runtime sees no - // replay preload on it and falls back to the run_started setup. - return outcome.result; - } const { canonicalEventId, maxEvents, events, cursor, hasMore } = outcome; const canonicalEvent = events.find( (event) => event.eventId === canonicalEventId diff --git a/packages/world-vercel/src/frames.test.ts b/packages/world-vercel/src/frames.test.ts index 72d312272e..036509052b 100644 --- a/packages/world-vercel/src/frames.test.ts +++ b/packages/world-vercel/src/frames.test.ts @@ -4,6 +4,8 @@ import { type DecodedFrame, decodeFrames, encodeFrame, + IncompleteFrameError, + InvalidFrameError, V4_FRAME_CONTENT_TYPE, } from './frames.js'; @@ -174,7 +176,36 @@ describe('decodeFrames', () => { it('throws when the stream ends mid-frame', async () => { const partial = encodeFrame({ x: 1 }, new Uint8Array(100)).slice(0, 20); const stream = streamOf(partial, 1024); - await expect(drainFrames(stream)).rejects.toThrow(/truncated/); + await expect(drainFrames(stream)).rejects.toBeInstanceOf( + IncompleteFrameError + ); + }); + + it('distinguishes invalid CBOR metadata from an incomplete response', async () => { + const invalidMeta = new Uint8Array([0, 0, 0, 1, 0x01, 0, 0, 0, 0]); + + await expect(drainFrames(streamOf(invalidMeta, 9))).rejects.toBeInstanceOf( + InvalidFrameError + ); + }); + + it('classifies a source read failure as an incomplete response', async () => { + const cause = new Error('socket reset'); + async function* failingSource(): AsyncGenerator { + yield new Uint8Array([0, 0]); + throw cause; + } + + await expect( + (async () => { + for await (const _frame of decodeFrames(failingSource())) { + // drain + } + })() + ).rejects.toMatchObject({ + name: 'IncompleteFrameError', + cause, + }); }); it('preserves CBOR types in meta (numbers, booleans, arrays)', async () => { diff --git a/packages/world-vercel/src/frames.ts b/packages/world-vercel/src/frames.ts index f5de347979..8fdffd976f 100644 --- a/packages/world-vercel/src/frames.ts +++ b/packages/world-vercel/src/frames.ts @@ -18,10 +18,48 @@ export interface DecodedFrame { body: Uint8Array; } +/** The response body stopped before the next complete frame was available. */ +export class IncompleteFrameError extends Error { + constructor(message: string, options?: ErrorOptions) { + super(message, options); + this.name = 'IncompleteFrameError'; + } +} + +/** The body was complete enough to decode, but did not contain valid frame metadata. */ +export class InvalidFrameError extends Error { + constructor(message: string, options?: ErrorOptions) { + super(message, options); + this.name = 'InvalidFrameError'; + } +} + // The protocol consumer validates the event or control-frame shape after the // body is available. The byte codec only requires a CBOR object here. const CborObjectSchema = z.record(z.string(), z.unknown()); +async function readChunk( + chunks: AsyncIterator +): Promise> { + try { + return await chunks.next(); + } catch (cause) { + throw new IncompleteFrameError('decodeFrames: source stream failed', { + cause, + }); + } +} + +function decodeFrameMeta(bytes: Uint8Array): Record { + try { + return CborObjectSchema.parse(decode(bytes)); + } catch (cause) { + throw new InvalidFrameError('decodeFrames: invalid CBOR metadata', { + cause, + }); + } +} + /** Test/utility: encode a complete frame. Production server uses prefix * + streaming body. */ export function encodeFrame( @@ -71,7 +109,7 @@ export async function* decodeFrames( const parts: Uint8Array[] = [buffer]; let byteLength = buffer.byteLength; while (byteLength < needed) { - const chunk = await chunks.next(); + const chunk = await readChunk(chunks); if (chunk.done) return false; if (chunk.value.byteLength === 0) continue; parts.push(chunk.value); @@ -104,12 +142,12 @@ export async function* decodeFrames( take(4); if (!(await refill(metaLen))) { - throw new Error('decodeFrames: truncated meta block'); + throw new IncompleteFrameError('decodeFrames: truncated meta block'); } - const meta = CborObjectSchema.parse(decode(take(metaLen))); + const meta = decodeFrameMeta(take(metaLen)); if (!(await refill(4))) { - throw new Error('decodeFrames: truncated body length'); + throw new IncompleteFrameError('decodeFrames: truncated body length'); } const bodyLen = new DataView( buffer.buffer, @@ -119,7 +157,7 @@ export async function* decodeFrames( take(4); if (bodyLen > 0 && !(await refill(bodyLen))) { - throw new Error('decodeFrames: truncated body bytes'); + throw new IncompleteFrameError('decodeFrames: truncated body bytes'); } // Slice (not subarray) so the yielded body owns its bytes — later // reads into the buffer won't overwrite it; bodyLen 0 yields empty. diff --git a/packages/world-vercel/src/http-core.ts b/packages/world-vercel/src/http-core.ts index c0729fbeb1..cc5c2086c4 100644 --- a/packages/world-vercel/src/http-core.ts +++ b/packages/world-vercel/src/http-core.ts @@ -604,7 +604,7 @@ export async function instrumentedFetch( throw error; } const ms = Date.now() - start; - if (!deferTransportSuccess || !response.ok) { + if (!deferTransportSuccess) { onTransportOutcome?.(); } diff --git a/packages/world/src/events.ts b/packages/world/src/events.ts index 90279e298c..defec5e0f4 100644 --- a/packages/world/src/events.ts +++ b/packages/world/src/events.ts @@ -941,11 +941,6 @@ export interface CreateEventParams { * `resumeHook()` must not set it. */ preloadEvents?: true; - /** - * Observe replay-preload events as their frames are decoded. This is a - * client-side delivery hook only; it is never serialized to a backend. - */ - onEvent?: (event: Event) => void; } /** @@ -1133,11 +1128,6 @@ export interface ListEventsParams { /** Omit `limit` to return every remaining event. */ pagination?: PaginationOptions; resolveData?: ResolveData; - /** - * Observe events as a streaming World decodes them. The callback runs - * synchronously and therefore applies response-stream backpressure. - */ - onEvent?: (event: Event) => void; } export interface ListEventsByCorrelationIdParams {